Vendor import of clang trunk r135360:

http://llvm.org/svn/llvm-project/cfe/trunk@135360

12 files changed, 378 insertions, 101 deletions

diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp
index bb43881..86ab956 100644
--- a/lib/Lex/HeaderSearch.cpp
+++ b/lib/Lex/HeaderSearch.cpp
@@ -123,10 +123,8 @@ const FileEntry *DirectoryLookup::LookupFile(
   llvm::SmallString<1024> TmpDir;
   if (isNormalDir()) {
     // Concatenate the requested file onto the directory.
-    // FIXME: Portability.  Filename concatenation should be in sys::Path.
-    TmpDir += getDir()->getName();
-    TmpDir.push_back('/');
-    TmpDir.append(Filename.begin(), Filename.end());
+    TmpDir = getDir()->getName();
+    llvm::sys::path::append(TmpDir, Filename);
     if (SearchPath != NULL) {
       llvm::StringRef SearchPathRef(getDir()->getName());
       SearchPath->clear();
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 3b1149c..a28b8f6 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -156,7 +156,7 @@ Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
 ///
 /// On entrance to this routine, TokStartLoc is a macro location which has a
 /// spelling loc that indicates the bytes to be lexed for the token and an
-/// instantiation location that indicates where all lexed tokens should be
+/// expansion location that indicates where all lexed tokens should be
 /// "expanded from".
 ///
 /// FIXME: It would really be nice to make _Pragma just be a wrapper around a
@@ -166,8 +166,8 @@ Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile,
 /// out of the critical path of the lexer!
 ///
 Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
-                                 SourceLocation InstantiationLocStart,
-                                 SourceLocation InstantiationLocEnd,
+                                 SourceLocation ExpansionLocStart,
+                                 SourceLocation ExpansionLocEnd,
                                  unsigned TokLen, Preprocessor &PP) {
   SourceManager &SM = PP.getSourceManager();
 
@@ -188,8 +188,8 @@ Lexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc,
   // Set the SourceLocation with the remapping information.  This ensures that
   // GetMappedTokenLoc will remap the tokens as they are lexed.
   L->FileLoc = SM.createInstantiationLoc(SM.getLocForStartOfFile(SpellingFID),
-                                         InstantiationLocStart,
-                                         InstantiationLocEnd, TokLen);
+                                         ExpansionLocStart,
+                                         ExpansionLocEnd, TokLen);
 
   // Ensure that the lexer thinks it is inside a directive, so that end \n will
   // return an EOD token.
@@ -621,7 +621,7 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
                                               unsigned CharNo,
                                               const SourceManager &SM,
                                               const LangOptions &Features) {
-  // Figure out how many physical characters away the specified instantiation
+  // Figure out how many physical characters away the specified expansion
   // character is.  This needs to take into consideration newlines and
   // trigraphs.
   bool Invalid = false;
@@ -679,9 +679,17 @@ SourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart,
 SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
                                           const SourceManager &SM,
                                           const LangOptions &Features) {
-  if (Loc.isInvalid() || !Loc.isFileID())
+  if (Loc.isInvalid())
     return SourceLocation();
-  
+
+  if (Loc.isMacroID()) {
+    if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, Features))
+      return SourceLocation(); // Points inside the macro expansion.
+
+    // Continue and find the location just after the macro expansion.
+    Loc = SM.getInstantiationRange(Loc).second;
+  }
+
   unsigned Len = Lexer::MeasureTokenLength(Loc, SM, Features);
   if (Len > Offset)
     Len = Len - Offset;
@@ -691,6 +699,58 @@ SourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset,
   return Loc.getFileLocWithOffset(Len);
 }
 
+/// \brief Returns true if the given MacroID location points at the first
+/// token of the macro expansion.
+bool Lexer::isAtStartOfMacroExpansion(SourceLocation loc,
+                                          const SourceManager &SM,
+                                          const LangOptions &LangOpts) {
+  assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
+
+  std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc);
+  // FIXME: If the token comes from the macro token paste operator ('##')
+  // this function will always return false;
+  if (infoLoc.second > 0)
+    return false; // Does not point at the start of token.
+
+  SourceLocation expansionLoc =
+    SM.getSLocEntry(infoLoc.first)
+      .getInstantiation().getInstantiationLocStart();
+  if (expansionLoc.isFileID())
+    return true; // No other macro expansions, this is the first.
+
+  return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts);
+}
+
+/// \brief Returns true if the given MacroID location points at the last
+/// token of the macro expansion.
+bool Lexer::isAtEndOfMacroExpansion(SourceLocation loc,
+                                        const SourceManager &SM,
+                                        const LangOptions &LangOpts) {
+  assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc");
+
+  SourceLocation spellLoc = SM.getSpellingLoc(loc);
+  unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts);
+  if (tokLen == 0)
+    return false;
+
+  FileID FID = SM.getFileID(loc);
+  SourceLocation afterLoc = loc.getFileLocWithOffset(tokLen+1);
+  if (!SM.isBeforeInSourceLocationOffset(afterLoc, SM.getNextOffset()))
+    return true; // We got past the last FileID, this points to the last token.
+
+  // FIXME: If the token comes from the macro token paste operator ('##')
+  // or the stringify operator ('#') this function will always return false;
+  if (FID == SM.getFileID(afterLoc))
+    return false; // Still in the same FileID, does not point to the last token.
+  
+  SourceLocation expansionLoc =
+    SM.getSLocEntry(FID).getInstantiation().getInstantiationLocEnd();
+  if (expansionLoc.isFileID())
+    return true; // No other macro expansions.
+
+  return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts);
+}
+
 //===----------------------------------------------------------------------===//
 // Character information.
 //===----------------------------------------------------------------------===//
@@ -829,7 +889,7 @@ static inline bool isNumberBody(unsigned char c) {
 //===----------------------------------------------------------------------===//
 
 /// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the
-/// lexer buffer was all instantiated at a single point, perform the mapping.
+/// lexer buffer was all expanded at a single point, perform the mapping.
 /// This is currently only used for _Pragma implementation, so it is the slow
 /// path of the hot getSourceLocation method.  Do not allow it to be inlined.
 static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
@@ -837,14 +897,14 @@ static LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc(
 static SourceLocation GetMappedTokenLoc(Preprocessor &PP,
                                         SourceLocation FileLoc,
                                         unsigned CharNo, unsigned TokLen) {
-  assert(FileLoc.isMacroID() && "Must be an instantiation");
+  assert(FileLoc.isMacroID() && "Must be a macro expansion");
 
   // Otherwise, we're lexing "mapped tokens".  This is used for things like
-  // _Pragma handling.  Combine the instantiation location of FileLoc with the
+  // _Pragma handling.  Combine the expansion location of FileLoc with the
   // spelling location.
   SourceManager &SM = PP.getSourceManager();
 
-  // Create a new SLoc which is expanded from Instantiation(FileLoc) but whose
+  // Create a new SLoc which is expanded from Expansion(FileLoc) but whose
   // characters come from spelling(FileLoc)+Offset.
   SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc);
   SpellingLoc = SpellingLoc.getFileLocWithOffset(CharNo);
diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp
index dee7da3..968c15e 100644
--- a/lib/Lex/MacroArgs.cpp
+++ b/lib/Lex/MacroArgs.cpp
@@ -185,7 +185,8 @@ MacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI,
 /// a character literal for the Microsoft charize (#@) extension.
 ///
 Token MacroArgs::StringifyArgument(const Token *ArgToks,
-                                   Preprocessor &PP, bool Charify) {
+                                   Preprocessor &PP, bool Charify,
+                                   SourceLocation hashInstLoc) {
   Token Tok;
   Tok.startToken();
   Tok.setKind(Charify ? tok::char_constant : tok::string_literal);
@@ -273,14 +274,15 @@ Token MacroArgs::StringifyArgument(const Token *ArgToks,
     }
   }
 
-  PP.CreateString(&Result[0], Result.size(), Tok);
+  PP.CreateString(&Result[0], Result.size(), Tok, hashInstLoc);
   return Tok;
 }
 
 /// getStringifiedArgument - Compute, cache, and return the specified argument
 /// that has been 'stringified' as required by the # operator.
 const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
-                                               Preprocessor &PP) {
+                                               Preprocessor &PP,
+                                               SourceLocation hashInstLoc) {
   assert(ArgNo < NumUnexpArgTokens && "Invalid argument number!");
   if (StringifiedArgs.empty()) {
     StringifiedArgs.resize(getNumArguments());
@@ -288,6 +290,7 @@ const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
            sizeof(StringifiedArgs[0])*getNumArguments());
   }
   if (StringifiedArgs[ArgNo].isNot(tok::string_literal))
-    StringifiedArgs[ArgNo] = StringifyArgument(getUnexpArgument(ArgNo), PP);
+    StringifiedArgs[ArgNo] = StringifyArgument(getUnexpArgument(ArgNo), PP,
+                                               /*Charify=*/false, hashInstLoc);
   return StringifiedArgs[ArgNo];
 }
diff --git a/lib/Lex/MacroArgs.h b/lib/Lex/MacroArgs.h
index 6ff4856..a962dac 100644
--- a/lib/Lex/MacroArgs.h
+++ b/lib/Lex/MacroArgs.h
@@ -20,6 +20,7 @@ namespace clang {
   class MacroInfo;
   class Preprocessor;
   class Token;
+  class SourceLocation;
 
 /// MacroArgs - An instance of this class captures information about
 /// the formal arguments specified to a function-like macro invocation.
@@ -86,7 +87,8 @@ public:
 
   /// getStringifiedArgument - Compute, cache, and return the specified argument
   /// that has been 'stringified' as required by the # operator.
-  const Token &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP);
+  const Token &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP,
+                                      SourceLocation hashInstLoc);
 
   /// getNumArguments - Return the number of arguments passed into this macro
   /// invocation.
@@ -106,7 +108,8 @@ public:
   /// a character literal for the Microsoft charize (#@) extension.
   ///
   static Token StringifyArgument(const Token *ArgToks,
-                                 Preprocessor &PP, bool Charify = false);
+                                 Preprocessor &PP, bool Charify,
+                                 SourceLocation hashInstLoc);
   
   
   /// deallocate - This should only be called by the Preprocessor when managing
diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp
index 66d87a1..0a16a25 100644
--- a/lib/Lex/MacroInfo.cpp
+++ b/lib/Lex/MacroInfo.cpp
@@ -25,6 +25,7 @@ MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc) {
   IsUsed = false;
   IsAllowRedefinitionsWithoutWarning = false;
   IsWarnIfUnused = false;
+  IsDefinitionLengthCached = false;
 
   ArgumentList = 0;
   NumArguments = 0;
@@ -43,11 +44,42 @@ MacroInfo::MacroInfo(const MacroInfo &MI, llvm::BumpPtrAllocator &PPAllocator) {
   IsUsed = MI.IsUsed;
   IsAllowRedefinitionsWithoutWarning = MI.IsAllowRedefinitionsWithoutWarning;
   IsWarnIfUnused = MI.IsWarnIfUnused;
+  IsDefinitionLengthCached = MI.IsDefinitionLengthCached;
+  DefinitionLength = MI.DefinitionLength;
   ArgumentList = 0;
   NumArguments = 0;
   setArgumentList(MI.ArgumentList, MI.NumArguments, PPAllocator);
 }
 
+unsigned MacroInfo::getDefinitionLengthSlow(SourceManager &SM) const {
+  assert(!IsDefinitionLengthCached);
+  IsDefinitionLengthCached = true;
+
+  if (ReplacementTokens.empty())
+    return (DefinitionLength = 0);
+
+  const Token &firstToken = ReplacementTokens.front();
+  const Token &lastToken = ReplacementTokens.back();
+  SourceLocation macroStart = firstToken.getLocation();
+  SourceLocation macroEnd = lastToken.getLocation();
+  assert(macroStart.isValid() && macroEnd.isValid());
+  assert((macroStart.isFileID() || firstToken.is(tok::comment)) &&
+         "Macro defined in macro?");
+  assert((macroEnd.isFileID() || lastToken.is(tok::comment)) &&
+         "Macro defined in macro?");
+  std::pair<FileID, unsigned>
+      startInfo = SM.getDecomposedInstantiationLoc(macroStart);
+  std::pair<FileID, unsigned>
+      endInfo = SM.getDecomposedInstantiationLoc(macroEnd);
+  assert(startInfo.first == endInfo.first &&
+         "Macro definition spanning multiple FileIDs ?");
+  assert(startInfo.second <= endInfo.second);
+  DefinitionLength = endInfo.second - startInfo.second;
+  DefinitionLength += lastToken.getLength();
+
+  return DefinitionLength;
+}
+
 /// isIdenticalTo - Return true if the specified macro definition is equal to
 /// this macro in spelling, arguments, and whitespace.  This is used to emit
 /// duplicate definition warnings.  This implements the rules in C99 6.10.3.
diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index 66e44bb..4af5fab 100644
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -784,8 +784,7 @@ void Preprocessor::HandleLineDirective(Token &Tok) {
       Diag(StrTok, diag::err_pp_linemarker_invalid_filename);
       return DiscardUntilEndOfDirective();
     }
-    FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString(),
-                                                  Literal.GetStringLength());
+    FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString());
 
     // Verify that there is nothing after the string, other than EOD.  Because
     // of C99 6.10.4p5, macros that expand to empty tokens are ok.
@@ -918,8 +917,7 @@ void Preprocessor::HandleDigitDirective(Token &DigitTok) {
       Diag(StrTok, diag::err_pp_linemarker_invalid_filename);
       return DiscardUntilEndOfDirective();
     }
-    FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString(),
-                                                  Literal.GetStringLength());
+    FilenameID = SourceMgr.getLineTableFilenameID(Literal.GetString());
 
     // If a filename was present, read any flags that are present.
     if (ReadLineMarkerFlags(IsFileEntry, IsFileExit,
@@ -1182,16 +1180,17 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
   const FileEntry *File = LookupFile(
       Filename, isAngled, LookupFrom, CurDir,
       Callbacks ? &SearchPath : NULL, Callbacks ? &RelativePath : NULL);
-  if (File == 0) {
-    Diag(FilenameTok, diag::err_pp_file_not_found) << Filename;
-    return;
-  }
 
   // Notify the callback object that we've seen an inclusion directive.
   if (Callbacks)
     Callbacks->InclusionDirective(HashLoc, IncludeTok, Filename, isAngled, File,
                                   End, SearchPath, RelativePath);
 
+  if (File == 0) {
+    Diag(FilenameTok, diag::warn_pp_file_not_found) << Filename;
+    return;
+  }
+
   // The #included file will be considered to be a system header if either it is
   // in a system include directory, or if the #includer is a system include
   // header.
@@ -1210,10 +1209,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
   // Look up the file, create a File ID for it.
   FileID FID = SourceMgr.createFileID(File, FilenameTok.getLocation(),
                                       FileCharacter);
-  if (FID.isInvalid()) {
-    Diag(FilenameTok, diag::err_pp_file_not_found) << Filename;
-    return;
-  }
+  assert(!FID.isInvalid() && "Expected valid file ID");
 
   // Finally, if all is good, enter the new file!
   EnterSourceFile(FID, CurDir, FilenameTok.getLocation());
@@ -1599,7 +1595,7 @@ void Preprocessor::HandleUndefDirective(Token &UndefTok) {
   // If the macro is not defined, this is a noop undef, just return.
   if (MI == 0) return;
 
-  if (!MI->isUsed())
+  if (!MI->isUsed() && MI->isWarnIfUnused())
     Diag(MI->getDefinitionLoc(), diag::pp_macro_not_used);
 
   // If the callbacks want to know, tell them about the macro #undef.
diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index bf0a7fb..bf28199 100644
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -265,6 +265,10 @@ bool Preprocessor::HandleEndOfTokenLexer(Token &Result) {
   assert(CurTokenLexer && !CurPPLexer &&
          "Ending a macro when currently in a #include file!");
 
+  if (!MacroExpandingLexersStack.empty() &&
+      MacroExpandingLexersStack.back().first == CurTokenLexer.get())
+    removeCachedMacroExpandedTokensOfLastLexer();
+
   // Delete or cache the now-dead macro expander.
   if (NumCachedTokenLexers == TokenLexerCacheSize)
     CurTokenLexer.reset();
diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp
index 01cd75f..ecd4d4c 100644
--- a/lib/Lex/PPMacroExpansion.cpp
+++ b/lib/Lex/PPMacroExpansion.cpp
@@ -22,6 +22,7 @@
 #include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Lex/ExternalPreprocessorSource.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Config/config.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cstdio>
@@ -194,9 +195,9 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
   /// invocation.
   MacroArgs *Args = 0;
 
-  // Remember where the end of the instantiation occurred.  For an object-like
+  // Remember where the end of the expansion occurred.  For an object-like
   // macro, this is the identifier.  For a function-like macro, this is the ')'.
-  SourceLocation InstantiationEnd = Identifier.getLocation();
+  SourceLocation ExpansionEnd = Identifier.getLocation();
 
   // If this is a function-like macro, read the arguments.
   if (MI->isFunctionLike()) {
@@ -209,7 +210,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
     // Preprocessor directives used inside macro arguments are not portable, and
     // this enables the warning.
     InMacroArgs = true;
-    Args = ReadFunctionLikeMacroArgs(Identifier, MI, InstantiationEnd);
+    Args = ReadFunctionLikeMacroArgs(Identifier, MI, ExpansionEnd);
 
     // Finished parsing args.
     InMacroArgs = false;
@@ -229,8 +230,8 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
   
   // If we started lexing a macro, enter the macro expansion body.
 
-  // Remember where the token is instantiated.
-  SourceLocation InstantiateLoc = Identifier.getLocation();
+  // Remember where the token is expanded.
+  SourceLocation ExpandLoc = Identifier.getLocation();
 
   // If this macro expands to no tokens, don't bother to push it onto the
   // expansion stack, only to take it right back off.
@@ -254,7 +255,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
       if (HadLeadingSpace) Identifier.setFlag(Token::LeadingSpace);
     }
     Identifier.setFlag(Token::LeadingEmptyMacro);
-    LastEmptyMacroInstantiationLoc = InstantiateLoc;
+    LastEmptyMacroExpansionLoc = ExpandLoc;
     ++NumFastMacroExpanded;
     return false;
 
@@ -280,11 +281,11 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
     Identifier.setFlagValue(Token::StartOfLine , isAtStartOfLine);
     Identifier.setFlagValue(Token::LeadingSpace, hasLeadingSpace);
 
-    // Update the tokens location to include both its instantiation and physical
+    // Update the tokens location to include both its expansion and physical
     // locations.
     SourceLocation Loc =
-      SourceMgr.createInstantiationLoc(Identifier.getLocation(), InstantiateLoc,
-                                       InstantiationEnd,Identifier.getLength());
+      SourceMgr.createInstantiationLoc(Identifier.getLocation(), ExpandLoc,
+                                       ExpansionEnd,Identifier.getLength());
     Identifier.setLocation(Loc);
 
     // If this is a disabled macro or #define X X, we must mark the result as
@@ -302,7 +303,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
   }
 
   // Start expanding the macro.
-  EnterMacro(Identifier, InstantiationEnd, Args);
+  EnterMacro(Identifier, ExpansionEnd, Args);
 
   // Now that the macro is at the top of the include stack, ask the
   // preprocessor to read the next token from it.
@@ -490,6 +491,46 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
                            isVarargsElided, *this);
 }
 
+/// \brief Keeps macro expanded tokens for TokenLexers.
+//
+/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
+/// going to lex in the cache and when it finishes the tokens are removed
+/// from the end of the cache.
+Token *Preprocessor::cacheMacroExpandedTokens(TokenLexer *tokLexer,
+                                              llvm::ArrayRef<Token> tokens) {
+  assert(tokLexer);
+  if (tokens.empty())
+    return 0;
+
+  size_t newIndex = MacroExpandedTokens.size();
+  bool cacheNeedsToGrow = tokens.size() >
+                      MacroExpandedTokens.capacity()-MacroExpandedTokens.size(); 
+  MacroExpandedTokens.append(tokens.begin(), tokens.end());
+
+  if (cacheNeedsToGrow) {
+    // Go through all the TokenLexers whose 'Tokens' pointer points in the
+    // buffer and update the pointers to the (potential) new buffer array.
+    for (unsigned i = 0, e = MacroExpandingLexersStack.size(); i != e; ++i) {
+      TokenLexer *prevLexer;
+      size_t tokIndex;
+      llvm::tie(prevLexer, tokIndex) = MacroExpandingLexersStack[i];
+      prevLexer->Tokens = MacroExpandedTokens.data() + tokIndex;
+    }
+  }
+
+  MacroExpandingLexersStack.push_back(std::make_pair(tokLexer, newIndex));
+  return MacroExpandedTokens.data() + newIndex;
+}
+
+void Preprocessor::removeCachedMacroExpandedTokensOfLastLexer() {
+  assert(!MacroExpandingLexersStack.empty());
+  size_t tokIndex = MacroExpandingLexersStack.back().second;
+  assert(tokIndex < MacroExpandedTokens.size());
+  // Pop the cached macro expanded tokens from the end.
+  MacroExpandedTokens.resize(tokIndex);
+  MacroExpandingLexersStack.pop_back();
+}
+
 /// ComputeDATE_TIME - Compute the current time, enter it into the specified
 /// scratch buffer, then return DATELoc/TIMELoc locations with the position of
 /// the identifier tokens inserted.
@@ -551,6 +592,11 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
            .Case("cxx_exceptions", LangOpts.Exceptions)
            .Case("cxx_rtti", LangOpts.RTTI)
            .Case("enumerator_attributes", true)
+           // Objective-C features
+           .Case("objc_arr", LangOpts.ObjCAutoRefCount) // FIXME: REMOVE?
+           .Case("objc_arc", LangOpts.ObjCAutoRefCount)
+           .Case("objc_arc_weak", LangOpts.ObjCAutoRefCount && 
+                 LangOpts.ObjCRuntimeHasWeak)
            .Case("objc_nonfragile_abi", LangOpts.ObjCNonFragileABI)
            .Case("objc_weak_class", LangOpts.ObjCNonFragileABI)
            .Case("ownership_holds", true)
@@ -787,10 +833,10 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     Loc = AdvanceToTokenCharacter(Loc, 0);
 
     // One wrinkle here is that GCC expands __LINE__ to location of the *end* of
-    // a macro instantiation.  This doesn't matter for object-like macros, but
+    // a macro expansion.  This doesn't matter for object-like macros, but
     // can matter for a function-like macro that expands to contain __LINE__.
-    // Skip down through instantiation points until we find a file loc for the
-    // end of the instantiation history.
+    // Skip down through expansion points until we find a file loc for the
+    // end of the expansion history.
     Loc = SourceMgr.getInstantiationRange(Loc).second;
     PresumedLoc PLoc = SourceMgr.getPresumedLoc(Loc);
 
diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp
index 23855d4..e6b28c1 100644
--- a/lib/Lex/Pragma.cpp
+++ b/lib/Lex/Pragma.cpp
@@ -326,9 +326,7 @@ void Preprocessor::HandlePragmaSystemHeader(Token &SysHeaderTok) {
   if (PLoc.isInvalid())
     return;
   
-  unsigned FilenameLen = strlen(PLoc.getFilename());
-  unsigned FilenameID = SourceMgr.getLineTableFilenameID(PLoc.getFilename(),
-                                                         FilenameLen);
+  unsigned FilenameID = SourceMgr.getLineTableFilenameID(PLoc.getFilename());
 
   // Notify the client, if desired, that we are in a new source file.
   if (Callbacks)
@@ -370,7 +368,7 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) {
   const DirectoryLookup *CurDir;
   const FileEntry *File = LookupFile(Filename, isAngled, 0, CurDir, NULL, NULL);
   if (File == 0) {
-    Diag(FilenameTok, diag::err_pp_file_not_found) << Filename;
+    Diag(FilenameTok, diag::warn_pp_file_not_found) << Filename;
     return;
   }
 
@@ -454,8 +452,7 @@ void Preprocessor::HandlePragmaComment(Token &Tok) {
       return;
     }
 
-    ArgumentString = std::string(Literal.GetString(),
-                                 Literal.GetString()+Literal.GetStringLength());
+    ArgumentString = Literal.GetString();
   }
 
   // FIXME: If the kind is "compiler" warn if the string is present (it is
@@ -531,7 +528,7 @@ void Preprocessor::HandlePragmaMessage(Token &Tok) {
     return;
   }
 
-  llvm::StringRef MessageString(Literal.GetString(), Literal.GetStringLength());
+  llvm::StringRef MessageString(Literal.GetString());
 
   if (ExpectClosingParen) {
     if (Tok.isNot(tok::r_paren)) {
@@ -839,8 +836,11 @@ struct PragmaDebugHandler : public PragmaHandler {
 
 /// PragmaDiagnosticHandler - e.g. '#pragma GCC diagnostic ignored "-Wformat"'
 struct PragmaDiagnosticHandler : public PragmaHandler {
+private:
+  const char *Namespace;
 public:
-  explicit PragmaDiagnosticHandler() : PragmaHandler("diagnostic") {}
+  explicit PragmaDiagnosticHandler(const char *NS) :
+    PragmaHandler("diagnostic"), Namespace(NS) {}
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &DiagToken) {
     SourceLocation DiagLoc = DiagToken.getLocation();
@@ -851,6 +851,7 @@ public:
       return;
     }
     IdentifierInfo *II = Tok.getIdentifierInfo();
+    PPCallbacks *Callbacks = PP.getPPCallbacks();
 
     diag::Mapping Map;
     if (II->isStr("warning"))
@@ -864,10 +865,13 @@ public:
     else if (II->isStr("pop")) {
       if (!PP.getDiagnostics().popMappings(DiagLoc))
         PP.Diag(Tok, diag::warn_pragma_diagnostic_cannot_pop);
-
+      else if (Callbacks)
+        Callbacks->PragmaDiagnosticPop(DiagLoc, Namespace);
       return;
     } else if (II->isStr("push")) {
       PP.getDiagnostics().pushMappings(DiagLoc);
+      if (Callbacks)
+        Callbacks->PragmaDiagnosticPush(DiagLoc, Namespace);
       return;
     } else {
       PP.Diag(Tok, diag::warn_pragma_diagnostic_invalid);
@@ -906,7 +910,7 @@ public:
       return;
     }
 
-    llvm::StringRef WarningName(Literal.GetString(), Literal.GetStringLength());
+    llvm::StringRef WarningName(Literal.GetString());
 
     if (WarningName.size() < 3 || WarningName[0] != '-' ||
         WarningName[1] != 'W') {
@@ -919,6 +923,8 @@ public:
                                                       Map, DiagLoc))
       PP.Diag(StrToks[0].getLocation(),
               diag::warn_pragma_diagnostic_unknown_warning) << WarningName;
+    else if (Callbacks)
+      Callbacks->PragmaDiagnostic(DiagLoc, Namespace, Map, WarningName);
   }
 };
 
@@ -1013,13 +1019,13 @@ void Preprocessor::RegisterBuiltinPragmas() {
   AddPragmaHandler("GCC", new PragmaPoisonHandler());
   AddPragmaHandler("GCC", new PragmaSystemHeaderHandler());
   AddPragmaHandler("GCC", new PragmaDependencyHandler());
-  AddPragmaHandler("GCC", new PragmaDiagnosticHandler());
+  AddPragmaHandler("GCC", new PragmaDiagnosticHandler("GCC"));
   // #pragma clang ...
   AddPragmaHandler("clang", new PragmaPoisonHandler());
   AddPragmaHandler("clang", new PragmaSystemHeaderHandler());
   AddPragmaHandler("clang", new PragmaDebugHandler());
   AddPragmaHandler("clang", new PragmaDependencyHandler());
-  AddPragmaHandler("clang", new PragmaDiagnosticHandler());
+  AddPragmaHandler("clang", new PragmaDiagnosticHandler("clang"));
 
   AddPragmaHandler("STDC", new PragmaSTDC_FENV_ACCESSHandler());
   AddPragmaHandler("STDC", new PragmaSTDC_CX_LIMITED_RANGEHandler());
diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp
index 0c8d948..9f93ab0 100644
--- a/lib/Lex/PreprocessingRecord.cpp
+++ b/lib/Lex/PreprocessingRecord.cpp
@@ -45,8 +45,8 @@ void PreprocessingRecord::MaybeLoadPreallocatedEntities() const {
   ExternalSource->ReadPreprocessedEntities();
 }
 
-PreprocessingRecord::PreprocessingRecord(bool IncludeNestedMacroInstantiations)
-  : IncludeNestedMacroInstantiations(IncludeNestedMacroInstantiations),
+PreprocessingRecord::PreprocessingRecord(bool IncludeNestedMacroExpansions)
+  : IncludeNestedMacroExpansions(IncludeNestedMacroExpansions),
     ExternalSource(0), NumPreallocatedEntities(0), 
     LoadedPreallocatedEntities(false)
 {
@@ -121,14 +121,13 @@ MacroDefinition *PreprocessingRecord::findMacroDefinition(const MacroInfo *MI) {
 }
 
 void PreprocessingRecord::MacroExpands(const Token &Id, const MacroInfo* MI) {
-  if (!IncludeNestedMacroInstantiations && Id.getLocation().isMacroID())
+  if (!IncludeNestedMacroExpansions && Id.getLocation().isMacroID())
     return;
 
   if (MacroDefinition *Def = findMacroDefinition(MI))
     PreprocessedEntities.push_back(
-                       new (*this) MacroInstantiation(Id.getIdentifierInfo(),
-                                                      Id.getLocation(),
-                                                      Def));
+                       new (*this) MacroExpansion(Id.getIdentifierInfo(),
+                                                  Id.getLocation(), Def));
 }
 
 void PreprocessingRecord::MacroDefined(const Token &Id,
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index fdc18f8..e7aa286 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -118,6 +118,8 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
 
 Preprocessor::~Preprocessor() {
   assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
+  assert(MacroExpandingLexersStack.empty() && MacroExpandedTokens.empty() &&
+         "Preprocessor::HandleEndOfTokenLexer should have cleared those");
 
   while (!IncludeMacroStack.empty()) {
     delete IncludeMacroStack.back().TheLexer;
@@ -225,6 +227,10 @@ Preprocessor::macro_begin(bool IncludeExternalMacros) const {
   return Macros.begin();
 }
 
+size_t Preprocessor::getTotalMemory() const {
+  return BP.getTotalMemory() + MacroExpandedTokens.capacity()*sizeof(Token);
+}
+
 Preprocessor::macro_iterator
 Preprocessor::macro_end(bool IncludeExternalMacros) const {
   if (IncludeExternalMacros && ExternalSource &&
@@ -322,15 +328,15 @@ llvm::StringRef Preprocessor::getSpelling(const Token &Tok,
 /// location for it.  If specified, the source location provides a source
 /// location for the token.
 void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
-                                SourceLocation InstantiationLoc) {
+                                SourceLocation ExpansionLoc) {
   Tok.setLength(Len);
 
   const char *DestPtr;
   SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
 
-  if (InstantiationLoc.isValid())
-    Loc = SourceMgr.createInstantiationLoc(Loc, InstantiationLoc,
-                                           InstantiationLoc, Len);
+  if (ExpansionLoc.isValid())
+    Loc = SourceMgr.createInstantiationLoc(Loc, ExpansionLoc,
+                                           ExpansionLoc, Len);
   Tok.setLocation(Loc);
 
   // If this is a raw identifier or a literal token, set the pointer data.
@@ -528,10 +534,10 @@ CommentHandler::~CommentHandler() { }
 CodeCompletionHandler::~CodeCompletionHandler() { }
 
 void Preprocessor::createPreprocessingRecord(
-                                      bool IncludeNestedMacroInstantiations) {
+                                      bool IncludeNestedMacroExpansions) {
   if (Record)
     return;
   
-  Record = new PreprocessingRecord(IncludeNestedMacroInstantiations);
+  Record = new PreprocessingRecord(IncludeNestedMacroExpansions);
   addPPCallbacks(Record);
 }
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index 65aff0d..8ff82f1 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -23,7 +23,7 @@ using namespace clang;
 
 /// Create a TokenLexer for the specified macro with the specified actual
 /// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
-void TokenLexer::Init(Token &Tok, SourceLocation ILEnd, MacroArgs *Actuals) {
+void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroArgs *Actuals) {
   // If the client is reusing a TokenLexer, make sure to free any memory
   // associated with it.
   destroy();
@@ -32,14 +32,36 @@ void TokenLexer::Init(Token &Tok, SourceLocation ILEnd, MacroArgs *Actuals) {
   ActualArgs = Actuals;
   CurToken = 0;
 
-  InstantiateLocStart = Tok.getLocation();
-  InstantiateLocEnd = ILEnd;
+  ExpandLocStart = Tok.getLocation();
+  ExpandLocEnd = ELEnd;
   AtStartOfLine = Tok.isAtStartOfLine();
   HasLeadingSpace = Tok.hasLeadingSpace();
   Tokens = &*Macro->tokens_begin();
   OwnsTokens = false;
   DisableMacroExpansion = false;
   NumTokens = Macro->tokens_end()-Macro->tokens_begin();
+  MacroExpansionStart = SourceLocation();
+
+  SourceManager &SM = PP.getSourceManager();
+  MacroStartSLocOffset = SM.getNextOffset();
+
+  if (NumTokens > 0) {
+    assert(Tokens[0].getLocation().isValid());
+    assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
+           "Macro defined in macro?");
+    assert(ExpandLocStart.isValid());
+
+    // Reserve a source location entry chunk for the length of the macro
+    // definition. Tokens that get lexed directly from the definition will
+    // have their locations pointing inside this chunk. This is to avoid
+    // creating separate source location entries for each token.
+    SourceLocation macroStart = SM.getInstantiationLoc(Tokens[0].getLocation());
+    MacroDefStartInfo = SM.getDecomposedLoc(macroStart);
+    MacroExpansionStart = SM.createInstantiationLoc(macroStart,
+                                              ExpandLocStart,
+                                              ExpandLocEnd,
+                                              Macro->getDefinitionLength(SM));
+  }
 
   // If this is a function-like macro, expand the arguments and change
   // Tokens to point to the expanded tokens.
@@ -69,9 +91,10 @@ void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
   DisableMacroExpansion = disableMacroExpansion;
   NumTokens = NumToks;
   CurToken = 0;
-  InstantiateLocStart = InstantiateLocEnd = SourceLocation();
+  ExpandLocStart = ExpandLocEnd = SourceLocation();
   AtStartOfLine = false;
   HasLeadingSpace = false;
+  MacroExpansionStart = SourceLocation();
 
   // Set HasLeadingSpace/AtStartOfLine so that the first token will be
   // returned unmodified.
@@ -98,6 +121,8 @@ void TokenLexer::destroy() {
 /// Expand the arguments of a function-like macro so that we can quickly
 /// return preexpanded tokens from Tokens.
 void TokenLexer::ExpandFunctionArguments() {
+  SourceManager &SM = PP.getSourceManager();
+
   llvm::SmallVector<Token, 128> ResultToks;
 
   // Loop through 'Tokens', expanding them into ResultToks.  Keep
@@ -119,13 +144,19 @@ void TokenLexer::ExpandFunctionArguments() {
       int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo());
       assert(ArgNo != -1 && "Token following # is not an argument?");
 
+      SourceLocation hashInstLoc;
+      if(ExpandLocStart.isValid()) {
+        hashInstLoc = getMacroExpansionLocation(CurTok.getLocation());
+        assert(hashInstLoc.isValid() && "Expected '#' to come from definition");
+      }
+
       Token Res;
       if (CurTok.is(tok::hash))  // Stringify
-        Res = ActualArgs->getStringifiedArgument(ArgNo, PP);
+        Res = ActualArgs->getStringifiedArgument(ArgNo, PP, hashInstLoc);
       else {
         // 'charify': don't bother caching these.
         Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
-                                           PP, true);
+                                           PP, true, hashInstLoc);
       }
 
       // The stringified/charified string leading space flag gets set to match
@@ -185,6 +216,27 @@ void TokenLexer::ExpandFunctionArguments() {
         unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
         ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
 
+        // If the '##' came from expanding an argument, turn it into 'unknown'
+        // to avoid pasting.
+        for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {
+          Token &Tok = ResultToks[i];
+          if (Tok.is(tok::hashhash))
+            Tok.setKind(tok::unknown);
+        }
+
+        if(ExpandLocStart.isValid()) {
+          SourceLocation curInst =
+              getMacroExpansionLocation(CurTok.getLocation());
+          assert(curInst.isValid() &&
+                 "Expected arg identifier to come from definition");
+          for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) {
+            Token &Tok = ResultToks[i];
+            Tok.setLocation(SM.createMacroArgInstantiationLoc(Tok.getLocation(),
+                                                              curInst,
+                                                              Tok.getLength()));
+          }
+        }
+
         // If any tokens were substituted from the argument, the whitespace
         // before the first token should match the whitespace of the arg
         // identifier.
@@ -220,6 +272,29 @@ void TokenLexer::ExpandFunctionArguments() {
 
       ResultToks.append(ArgToks, ArgToks+NumToks);
 
+      // If the '##' came from expanding an argument, turn it into 'unknown'
+      // to avoid pasting.
+      for (unsigned i = ResultToks.size() - NumToks, e = ResultToks.size();
+             i != e; ++i) {
+        Token &Tok = ResultToks[i];
+        if (Tok.is(tok::hashhash))
+          Tok.setKind(tok::unknown);
+      }
+
+      if (ExpandLocStart.isValid()) {
+        SourceLocation curInst =
+            getMacroExpansionLocation(CurTok.getLocation());
+        assert(curInst.isValid() &&
+               "Expected arg identifier to come from definition");
+        for (unsigned i = ResultToks.size() - NumToks, e = ResultToks.size();
+               i != e; ++i) {
+          Token &Tok = ResultToks[i];
+          Tok.setLocation(SM.createMacroArgInstantiationLoc(Tok.getLocation(),
+                                                            curInst,
+                                                            Tok.getLength()));
+        }
+      }
+
       // If this token (the macro argument) was supposed to get leading
       // whitespace, transfer this information onto the first token of the
       // expansion.
@@ -284,15 +359,11 @@ void TokenLexer::ExpandFunctionArguments() {
     assert(!OwnsTokens && "This would leak if we already own the token list");
     // This is deleted in the dtor.
     NumTokens = ResultToks.size();
-    llvm::BumpPtrAllocator &Alloc = PP.getPreprocessorAllocator();
-    Token *Res =
-      static_cast<Token *>(Alloc.Allocate(sizeof(Token)*ResultToks.size(),
-                                          llvm::alignOf<Token>()));
-    if (NumTokens)
-      memcpy(Res, &ResultToks[0], NumTokens*sizeof(Token));
-    Tokens = Res;
-
-    // The preprocessor bump pointer owns these tokens, not us.
+    // The tokens will be added to Preprocessor's cache and will be removed
+    // when this TokenLexer finishes lexing them.
+    Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
+
+    // The preprocessor cache of macro expanded tokens owns these tokens,not us.
     OwnsTokens = false;
   }
 }
@@ -317,6 +388,8 @@ void TokenLexer::Lex(Token &Tok) {
     return PPCache.Lex(Tok);
   }
 
+  SourceManager &SM = PP.getSourceManager();
+
   // If this is the first token of the expanded result, we inherit spacing
   // properties later.
   bool isFirstToken = CurToken == 0;
@@ -327,7 +400,8 @@ void TokenLexer::Lex(Token &Tok) {
   bool TokenIsFromPaste = false;
 
   // If this token is followed by a token paste (##) operator, paste the tokens!
-  if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)) {
+  // Note that ## is a normal token when not expanding a macro.
+  if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash) && Macro) {
     // When handling the microsoft /##/ extension, the final token is
     // returned by PasteTokens, not the pasted token.
     if (PasteTokens(Tok))
@@ -339,14 +413,25 @@ void TokenLexer::Lex(Token &Tok) {
   // The token's current location indicate where the token was lexed from.  We
   // need this information to compute the spelling of the token, but any
   // diagnostics for the expanded token should appear as if they came from
-  // InstantiationLoc.  Pull this information together into a new SourceLocation
+  // ExpansionLoc.  Pull this information together into a new SourceLocation
   // that captures all of this.
-  if (InstantiateLocStart.isValid()) {   // Don't do this for token streams.
-    SourceManager &SM = PP.getSourceManager();
-    Tok.setLocation(SM.createInstantiationLoc(Tok.getLocation(),
-                                              InstantiateLocStart,
-                                              InstantiateLocEnd,
-                                              Tok.getLength()));
+  if (ExpandLocStart.isValid() &&   // Don't do this for token streams.
+      // Check that the token's location was not already set properly.
+      SM.isBeforeInSourceLocationOffset(Tok.getLocation(),
+                                        MacroStartSLocOffset)) {
+    SourceLocation instLoc;
+    if (Tok.is(tok::comment)) {
+      instLoc = SM.createInstantiationLoc(Tok.getLocation(),
+                                          ExpandLocStart,
+                                          ExpandLocEnd,
+                                          Tok.getLength());
+    } else {
+      instLoc = getMacroExpansionLocation(Tok.getLocation());
+      assert(instLoc.isValid() &&
+             "Location for token not coming from definition was not set!");
+    }
+
+    Tok.setLocation(instLoc);
   }
 
   // If this is the first token, set the lexical properties of the token to
@@ -384,9 +469,10 @@ void TokenLexer::Lex(Token &Tok) {
 bool TokenLexer::PasteTokens(Token &Tok) {
   llvm::SmallString<128> Buffer;
   const char *ResultTokStrPtr = 0;
+  SourceLocation PasteOpLoc;
   do {
     // Consume the ## operator.
-    SourceLocation PasteOpLoc = Tokens[CurToken].getLocation();
+    PasteOpLoc = Tokens[CurToken].getLocation();
     ++CurToken;
     assert(!isAtEnd() && "No token on the RHS of a paste operator!");
 
@@ -484,12 +570,12 @@ bool TokenLexer::PasteTokens(Token &Tok) {
 
         // Do not emit the error when preprocessing assembler code.
         if (!PP.getLangOptions().AsmPreprocessor) {
-          // Explicitly convert the token location to have proper instantiation
+          // Explicitly convert the token location to have proper expansion
           // information so that the user knows where it came from.
           SourceManager &SM = PP.getSourceManager();
           SourceLocation Loc =
-            SM.createInstantiationLoc(PasteOpLoc, InstantiateLocStart,
-                                      InstantiateLocEnd, 2);
+            SM.createInstantiationLoc(PasteOpLoc, ExpandLocStart,
+                                      ExpandLocEnd, 2);
           // If we're in microsoft extensions mode, downgrade this from a hard
           // error to a warning that defaults to an error.  This allows
           // disabling it.
@@ -512,12 +598,30 @@ bool TokenLexer::PasteTokens(Token &Tok) {
     // Transfer properties of the LHS over the the Result.
     Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine());
     Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace());
-
+    
     // Finally, replace LHS with the result, consume the RHS, and iterate.
     ++CurToken;
     Tok = Result;
   } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash));
 
+  // The token's current location indicate where the token was lexed from.  We
+  // need this information to compute the spelling of the token, but any
+  // diagnostics for the expanded token should appear as if the token was
+  // expanded from the (##) operator. Pull this information together into
+  // a new SourceLocation that captures all of this.
+  if (ExpandLocStart.isValid()) {
+    SourceManager &SM = PP.getSourceManager();
+    SourceLocation pasteLocInst =
+        getMacroExpansionLocation(PasteOpLoc);
+    assert(pasteLocInst.isValid() &&
+           "Expected '##' to come from definition");
+
+    Tok.setLocation(SM.createInstantiationLoc(Tok.getLocation(),
+                                              pasteLocInst,
+                                              pasteLocInst,
+                                              Tok.getLength()));
+  }
+
   // Now that we got the result token, it will be subject to expansion.  Since
   // token pasting re-lexes the result token in raw mode, identifier information
   // isn't looked up.  As such, if the result is an identifier, look up id info.
@@ -548,7 +652,7 @@ bool TokenLexer::isParsingPreprocessorDirective() const {
 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
 /// together to form a comment that comments out everything in the current
 /// macro, other active macros, and anything left on the current physical
-/// source line of the instantiated buffer.  Handle this by returning the
+/// source line of the expanded buffer.  Handle this by returning the
 /// first token on the next line.
 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) {
   // We 'comment out' the rest of this macro by just ignoring the rest of the
@@ -561,3 +665,23 @@ void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) {
 
   PP.HandleMicrosoftCommentPaste(Tok);
 }
+
+/// \brief If \arg loc is a FileID and points inside the current macro
+/// definition, returns the appropriate source location pointing at the
+/// macro expansion source location entry.
+SourceLocation TokenLexer::getMacroExpansionLocation(SourceLocation loc) const {
+  assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
+         "Not appropriate for token streams");
+  assert(loc.isValid());
+  
+  SourceManager &SM = PP.getSourceManager();
+  unsigned relativeOffset;
+  if (loc.isFileID() &&
+      SM.isInFileID(loc,
+                    MacroDefStartInfo.first, MacroDefStartInfo.second,
+                    Macro->getDefinitionLength(SM), &relativeOffset)) {
+    return MacroExpansionStart.getFileLocWithOffset(relativeOffset);
+  }
+
+  return SourceLocation();
+}