Vendor import of clang RELEASE_360/rc1 tag r226102 (effectively, 3.6.0 RC1):

https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_360/rc1@226102

1 files changed, 17 insertions, 3 deletions

diff --git a/lib/Lex/TokenConcatenation.cpp b/lib/Lex/TokenConcatenation.cpp
index 0a66bba..0832749 100644
--- a/lib/Lex/TokenConcatenation.cpp
+++ b/lib/Lex/TokenConcatenation.cpp
@@ -99,6 +99,10 @@ TokenConcatenation::TokenConcatenation(Preprocessor &pp) : PP(pp) {
     TokenInfo[tok::utf32_char_constant ] |= aci_custom;
   }
 
+  // These tokens have custom code in C++1z mode.
+  if (PP.getLangOpts().CPlusPlus1z)
+    TokenInfo[tok::utf8_char_constant] |= aci_custom;
+
   // These tokens change behavior if followed by an '='.
   TokenInfo[tok::amp         ] |= aci_avoid_equal;           // &=
   TokenInfo[tok::plus        ] |= aci_avoid_equal;           // +=
@@ -163,8 +167,8 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
     return false;
 
   tok::TokenKind PrevKind = PrevTok.getKind();
-  if (PrevTok.getIdentifierInfo())  // Language keyword or named operator.
-    PrevKind = tok::identifier;
+  if (!PrevTok.isAnnotation() && PrevTok.getIdentifierInfo())
+    PrevKind = tok::identifier; // Language keyword or named operator.
 
   // Look up information on when we should avoid concatenation with prevtok.
   unsigned ConcatInfo = TokenInfo[PrevKind];
@@ -178,6 +182,14 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
       return true;
     ConcatInfo &= ~aci_avoid_equal;
   }
+  if (Tok.isAnnotation()) {
+    // Modules annotation can show up when generated automatically for includes.
+    assert((Tok.is(tok::annot_module_include) ||
+            Tok.is(tok::annot_module_begin) ||
+            Tok.is(tok::annot_module_end)) &&
+           "unexpected annotation in AvoidConcat");
+    ConcatInfo = 0;
+  }
 
   if (ConcatInfo == 0) return false;
 
@@ -205,6 +217,7 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
   case tok::utf32_string_literal:
   case tok::char_constant:
   case tok::wide_char_constant:
+  case tok::utf8_char_constant:
   case tok::utf16_char_constant:
   case tok::utf32_char_constant:
     if (!PP.getLangOpts().CPlusPlus11)
@@ -228,7 +241,8 @@ bool TokenConcatenation::AvoidConcat(const Token &PrevPrevTok,
     if (Tok.getIdentifierInfo() || Tok.is(tok::wide_string_literal) ||
         Tok.is(tok::utf8_string_literal) || Tok.is(tok::utf16_string_literal) ||
         Tok.is(tok::utf32_string_literal) || Tok.is(tok::wide_char_constant) ||
-        Tok.is(tok::utf16_char_constant) || Tok.is(tok::utf32_char_constant))
+        Tok.is(tok::utf8_char_constant) || Tok.is(tok::utf16_char_constant) ||
+        Tok.is(tok::utf32_char_constant))
       return true;
 
     // If this isn't identifier + string, we're done.