2 files changed, 423 insertions, 126 deletions
diff --git a/tools/c-index-test/Makefile b/tools/c-index-test/Makefile
index 06e2405..e9ba174 100644
--- a/tools/c-index-test/Makefile
+++ b/tools/c-index-test/Makefile
@@ -10,7 +10,6 @@ LEVEL = ../../../..
 
 TOOLNAME = c-index-test
 CPPFLAGS += -I$(PROJ_SRC_DIR)/../../include -I$(PROJ_OBJ_DIR)/../../include
-CXXFLAGS = -fno-rtti
 NO_INSTALL = 1
 
 # No plugins, optimize startup time.
diff --git a/tools/c-index-test/c-index-test.c b/tools/c-index-test/c-index-test.c
index 4ef3904..d4771c7 100644
--- a/tools/c-index-test/c-index-test.c
+++ b/tools/c-index-test/c-index-test.c
@@ -28,10 +28,20 @@ char *basename(const char* path)
 extern char *basename(const char *);
 #endif
 
+static void PrintDiagnosticCallback(CXDiagnostic Diagnostic, 
+                                    CXClientData ClientData);
+
+
+static void PrintExtent(FILE *out, unsigned begin_line, unsigned begin_column,
+                        unsigned end_line, unsigned end_column) {
+  fprintf(out, "[%d:%d - %d:%d]", begin_line, begin_column,
+          end_line, end_column);
+}
+
 static unsigned CreateTranslationUnit(CXIndex Idx, const char *file,
                                       CXTranslationUnit *TU) {
   
-  *TU = clang_createTranslationUnit(Idx, file);
+  *TU = clang_createTranslationUnit(Idx, file, PrintDiagnosticCallback, stderr);
   if (!TU) {
     fprintf(stderr, "Unable to load translation unit from '%s'!\n", file);
     return 0;
@@ -151,7 +161,7 @@ static void PrintCursor(CXCursor Cursor) {
     Referenced = clang_getCursorReferenced(Cursor);
     if (!clang_equalCursors(Referenced, clang_getNullCursor())) {
       CXSourceLocation Loc = clang_getCursorLocation(Referenced);
-      clang_getInstantiationLocation(Loc, 0, &line, &column);
+      clang_getInstantiationLocation(Loc, 0, &line, &column, 0);
       printf(":%d:%d", line, column);
     }
 
@@ -164,7 +174,7 @@ static const char* GetCursorSource(CXCursor Cursor) {
   CXSourceLocation Loc = clang_getCursorLocation(Cursor);
   const char *source;
   CXFile file;
-  clang_getInstantiationLocation(Loc, &file, 0, 0);
+  clang_getInstantiationLocation(Loc, &file, 0, 0, 0);
   source = clang_getFileName(file);
   if (!source)
     return "<invalid loc>";  
@@ -172,6 +182,131 @@ static const char* GetCursorSource(CXCursor Cursor) {
 }
 
 /******************************************************************************/
+/* Callbacks.                                                                 */
+/******************************************************************************/
+
+typedef void (*PostVisitTU)(CXTranslationUnit);
+
+static void PrintDiagnosticCallback(CXDiagnostic Diagnostic, 
+                                    CXClientData ClientData) {
+  FILE *out = (FILE *)ClientData;
+  CXFile file;
+  unsigned line, column;
+  CXString text;
+  enum CXDiagnosticSeverity severity = clang_getDiagnosticSeverity(Diagnostic);
+  
+  /* Ignore diagnostics that should be ignored. */
+  if (severity == CXDiagnostic_Ignored)
+    return;
+  
+  /* Print file:line:column. */
+  clang_getInstantiationLocation(clang_getDiagnosticLocation(Diagnostic),
+                                 &file, &line, &column, 0);
+  if (file) {
+    unsigned i, n;
+    unsigned printed_any_ranges = 0;
+    
+    fprintf(out, "%s:%d:%d:", clang_getFileName(file), line, column);
+  
+    n = clang_getDiagnosticNumRanges(Diagnostic);
+    for (i = 0; i != n; ++i) {
+      CXFile start_file, end_file;
+      CXSourceRange range = clang_getDiagnosticRange(Diagnostic, i);
+      
+      unsigned start_line, start_column, end_line, end_column;
+      clang_getInstantiationLocation(clang_getRangeStart(range),
+                                     &start_file, &start_line, &start_column,0);
+      clang_getInstantiationLocation(clang_getRangeEnd(range),
+                                     &end_file, &end_line, &end_column, 0);
+      
+      if (start_file != end_file || start_file != file)
+        continue;
+
+      PrintExtent(out, start_line, start_column, end_line, end_column);
+      printed_any_ranges = 1;
+    }
+    if (printed_any_ranges)
+      fprintf(out, ":");
+    
+    fprintf(out, " ");
+  }
+  
+  /* Print warning/error/etc. */
+  switch (severity) {
+  case CXDiagnostic_Ignored: assert(0 && "impossible"); break;
+  case CXDiagnostic_Note: fprintf(out, "note: "); break;
+  case CXDiagnostic_Warning: fprintf(out, "warning: "); break;
+  case CXDiagnostic_Error: fprintf(out, "error: "); break;
+  case CXDiagnostic_Fatal: fprintf(out, "fatal error: "); break;
+  }
+  
+  text = clang_getDiagnosticSpelling(Diagnostic);
+  if (clang_getCString(text))
+    fprintf(out, "%s\n", clang_getCString(text));
+  else
+    fprintf(out, "<no diagnostic text>\n");
+  clang_disposeString(text);
+  
+  if (file) {
+    unsigned i, num_fixits = clang_getDiagnosticNumFixIts(Diagnostic);
+    for (i = 0; i != num_fixits; ++i) {
+      switch (clang_getDiagnosticFixItKind(Diagnostic, i)) {
+      case CXFixIt_Insertion: {
+        CXSourceLocation insertion_loc;
+        CXFile insertion_file;
+        unsigned insertion_line, insertion_column;
+        text = clang_getDiagnosticFixItInsertion(Diagnostic, i, &insertion_loc);
+        clang_getInstantiationLocation(insertion_loc, &insertion_file, 
+                                       &insertion_line, &insertion_column, 0);
+        if (insertion_file == file)
+          fprintf(out, "FIX-IT: Insert \"%s\" at %d:%d\n",
+                  clang_getCString(text), insertion_line, insertion_column);
+        clang_disposeString(text);
+        break;
+      }
+
+      case CXFixIt_Removal: {
+        CXFile start_file, end_file;
+        unsigned start_line, start_column, end_line, end_column;
+        CXSourceRange remove_range
+          = clang_getDiagnosticFixItRemoval(Diagnostic, i);
+        clang_getInstantiationLocation(clang_getRangeStart(remove_range),
+                                       &start_file, &start_line, &start_column,
+                                       0);
+        clang_getInstantiationLocation(clang_getRangeEnd(remove_range),
+                                       &end_file, &end_line, &end_column, 0);
+        if (start_file == file && end_file == file) {
+          fprintf(out, "FIX-IT: Remove ");
+          PrintExtent(out, start_line, start_column, end_line, end_column);
+          fprintf(out, "\n");
+        }
+        break;
+      }
+          
+      case CXFixIt_Replacement: {
+        CXFile start_file, end_file;
+        unsigned start_line, start_column, end_line, end_column;
+        CXSourceRange remove_range;
+        text = clang_getDiagnosticFixItReplacement(Diagnostic, i,&remove_range);
+        clang_getInstantiationLocation(clang_getRangeStart(remove_range),
+                                       &start_file, &start_line, &start_column,
+                                       0);
+        clang_getInstantiationLocation(clang_getRangeEnd(remove_range),
+                                       &end_file, &end_line, &end_column, 0);
+        if (start_file == end_file) {
+          fprintf(out, "FIX-IT: Replace ");
+          PrintExtent(out, start_line, start_column, end_line, end_column);
+          fprintf(out, " with \"%s\"\n", clang_getCString(text));
+        }
+        clang_disposeString(text);
+        break;
+      }
+      }
+    }
+  }
+}
+
+/******************************************************************************/
 /* Logic for testing traversal.                                               */
 /******************************************************************************/
 
@@ -183,14 +318,14 @@ static void PrintCursorExtent(CXCursor C) {
   unsigned begin_line, begin_column, end_line, end_column;
   
   clang_getInstantiationLocation(clang_getRangeStart(extent),
-                                 &begin_file, &begin_line, &begin_column);
+                                 &begin_file, &begin_line, &begin_column, 0);
   clang_getInstantiationLocation(clang_getRangeEnd(extent),
-                                 &end_file, &end_line, &end_column);
+                                 &end_file, &end_line, &end_column, 0);
   if (!begin_file || !end_file)
     return;
 
-  printf(" [Extent=%d:%d:%d:%d]", begin_line, begin_column,
-         end_line, end_column);
+  printf(" Extent=");
+  PrintExtent(stdout, begin_line, begin_column, end_line, end_column);
 }
 
 /* Data used by all of the visitors. */
@@ -207,7 +342,7 @@ enum CXChildVisitResult FilteredPrintingVisitor(CXCursor Cursor,
   if (!Data->Filter || (Cursor.kind == *(enum CXCursorKind *)Data->Filter)) {
     CXSourceLocation Loc = clang_getCursorLocation(Cursor);
     unsigned line, column;
-    clang_getInstantiationLocation(Loc, 0, &line, &column);
+    clang_getInstantiationLocation(Loc, 0, &line, &column, 0);
     printf("// %s: %s:%d:%d: ", FileCheckPrefix,
            GetCursorSource(Cursor), line, column);
     PrintCursor(Cursor);
@@ -251,7 +386,7 @@ static enum CXChildVisitResult FunctionScanVisitor(CXCursor Cursor,
       curColumn++;
           
     Loc = clang_getCursorLocation(Cursor);
-    clang_getInstantiationLocation(Loc, &file, 0, 0);
+    clang_getInstantiationLocation(Loc, &file, 0, 0, 0);
     source = clang_getFileName(file);
     if (source) {
       CXSourceLocation RefLoc
@@ -297,56 +432,93 @@ enum CXChildVisitResult USRVisitor(CXCursor C, CXCursor parent,
 }
 
 /******************************************************************************/
+/* Inclusion stack testing.                                                   */
+/******************************************************************************/
+
+void InclusionVisitor(CXFile includedFile, CXSourceLocation *includeStack,
+                      unsigned includeStackLen, CXClientData data) {
+  
+  unsigned i;
+  printf("file: %s\nincluded by:\n", clang_getFileName(includedFile));
+  for (i = 0; i < includeStackLen; ++i) {
+    CXFile includingFile;
+    unsigned line, column;
+    clang_getInstantiationLocation(includeStack[i], &includingFile, &line,
+                                   &column, 0);
+    printf("  %s:%d:%d\n", clang_getFileName(includingFile), line, column);
+  }
+  printf("\n");
+}
+
+void PrintInclusionStack(CXTranslationUnit TU) {
+  clang_getInclusions(TU, InclusionVisitor, NULL);  
+}
+
+/******************************************************************************/
 /* Loading ASTs/source.                                                       */
 /******************************************************************************/
 
 static int perform_test_load(CXIndex Idx, CXTranslationUnit TU,
                              const char *filter, const char *prefix,
-                             CXCursorVisitor Visitor) {
-  enum CXCursorKind K = CXCursor_NotImplemented;
-  enum CXCursorKind *ck = &K;
-  VisitorData Data;
+                             CXCursorVisitor Visitor,
+                             PostVisitTU PV) {
   
   if (prefix)
     FileCheckPrefix = prefix;  
+
+  if (Visitor) {
+    enum CXCursorKind K = CXCursor_NotImplemented;
+    enum CXCursorKind *ck = &K;
+    VisitorData Data;
   
-  /* Perform some simple filtering. */
-  if (!strcmp(filter, "all") || !strcmp(filter, "local")) ck = NULL;
-  else if (!strcmp(filter, "category")) K = CXCursor_ObjCCategoryDecl;
-  else if (!strcmp(filter, "interface")) K = CXCursor_ObjCInterfaceDecl;
-  else if (!strcmp(filter, "protocol")) K = CXCursor_ObjCProtocolDecl;
-  else if (!strcmp(filter, "function")) K = CXCursor_FunctionDecl;
-  else if (!strcmp(filter, "typedef")) K = CXCursor_TypedefDecl;
-  else if (!strcmp(filter, "scan-function")) Visitor = FunctionScanVisitor;
-  else {
-    fprintf(stderr, "Unknown filter for -test-load-tu: %s\n", filter);
-    return 1;
+    /* Perform some simple filtering. */
+    if (!strcmp(filter, "all") || !strcmp(filter, "local")) ck = NULL;
+    else if (!strcmp(filter, "none")) K = (enum CXCursorKind) ~0;
+    else if (!strcmp(filter, "category")) K = CXCursor_ObjCCategoryDecl;
+    else if (!strcmp(filter, "interface")) K = CXCursor_ObjCInterfaceDecl;
+    else if (!strcmp(filter, "protocol")) K = CXCursor_ObjCProtocolDecl;
+    else if (!strcmp(filter, "function")) K = CXCursor_FunctionDecl;
+    else if (!strcmp(filter, "typedef")) K = CXCursor_TypedefDecl;
+    else if (!strcmp(filter, "scan-function")) Visitor = FunctionScanVisitor;
+    else {
+      fprintf(stderr, "Unknown filter for -test-load-tu: %s\n", filter);
+      return 1;
+    }
+  
+    Data.TU = TU;
+    Data.Filter = ck;
+    clang_visitChildren(clang_getTranslationUnitCursor(TU), Visitor, &Data);
   }
   
-  Data.TU = TU;
-  Data.Filter = ck;
-  clang_visitChildren(clang_getTranslationUnitCursor(TU), Visitor, &Data);
+  if (PV)
+    PV(TU);
+
   clang_disposeTranslationUnit(TU);
   return 0;
 }
 
 int perform_test_load_tu(const char *file, const char *filter,
-                         const char *prefix,
-                         CXCursorVisitor Visitor) {
+                         const char *prefix, CXCursorVisitor Visitor,
+                         PostVisitTU PV) {
   CXIndex Idx;
   CXTranslationUnit TU;
+  int result;
   Idx = clang_createIndex(/* excludeDeclsFromPCH */ 
-                          !strcmp(filter, "local") ? 1 : 0, 
-                          /* displayDiagnostics */ 1);
+                          !strcmp(filter, "local") ? 1 : 0);
   
-  if (!CreateTranslationUnit(Idx, file, &TU))
+  if (!CreateTranslationUnit(Idx, file, &TU)) {
+    clang_disposeIndex(Idx);
     return 1;
+  }
 
-  return perform_test_load(Idx, TU, filter, prefix, Visitor);
+  result = perform_test_load(Idx, TU, filter, prefix, Visitor, PV);
+  clang_disposeIndex(Idx);
+  return result;
 }
 
-int perform_test_load_source(int argc, const char **argv, const char *filter,
-                             CXCursorVisitor Visitor) {
+int perform_test_load_source(int argc, const char **argv,
+                             const char *filter, CXCursorVisitor Visitor,
+                             PostVisitTU PV) {
   const char *UseExternalASTs =
     getenv("CINDEXTEST_USE_EXTERNAL_AST_GENERATION");
   CXIndex Idx;
@@ -356,27 +528,32 @@ int perform_test_load_source(int argc, const char **argv, const char *filter,
   int result;
   
   Idx = clang_createIndex(/* excludeDeclsFromPCH */
-                          !strcmp(filter, "local") ? 1 : 0,
-                          /* displayDiagnostics */ 1);
+                          !strcmp(filter, "local") ? 1 : 0);
 
   if (UseExternalASTs && strlen(UseExternalASTs))
     clang_setUseExternalASTGeneration(Idx, 1);
 
-  if (parse_remapped_files(argc, argv, 0, &unsaved_files, &num_unsaved_files))
+  if (parse_remapped_files(argc, argv, 0, &unsaved_files, &num_unsaved_files)) {
+    clang_disposeIndex(Idx);
     return -1;
+  }
 
   TU = clang_createTranslationUnitFromSourceFile(Idx, 0, 
                                                  argc - num_unsaved_files, 
                                                  argv + num_unsaved_files,
                                                  num_unsaved_files,
-                                                 unsaved_files);
+                                                 unsaved_files,
+                                                 PrintDiagnosticCallback,
+                                                 stderr);
   if (!TU) {
     fprintf(stderr, "Unable to load translation unit!\n");
+    clang_disposeIndex(Idx);
     return 1;
   }
 
-  result = perform_test_load(Idx, TU, filter, NULL, Visitor);
+  result = perform_test_load(Idx, TU, filter, NULL, Visitor, PV);
   free_remapped_files(unsaved_files, num_unsaved_files);
+  clang_disposeIndex(Idx);
   return result;
 }
 
@@ -391,8 +568,8 @@ static void print_cursor_file_scan(CXCursor cursor,
   printf("// %s: ", FileCheckPrefix);
   if (prefix)
     printf("-%s", prefix);
-  printf("{start_line=%d start_col=%d end_line=%d end_col=%d} ",
-          start_line, start_col, end_line, end_col);
+  PrintExtent(stdout, start_line, start_col, end_line, end_col);
+  printf(" ");
   PrintCursor(cursor);
   printf("\n");
 }
@@ -402,15 +579,12 @@ static int perform_file_scan(const char *ast_file, const char *source_file,
   CXIndex Idx;
   CXTranslationUnit TU;
   FILE *fp;
-  unsigned line;
-  CXCursor prevCursor;
+  CXCursor prevCursor = clang_getNullCursor();
   CXFile file;
-  unsigned printed;
-  unsigned start_line, start_col, last_line, last_col;
-  size_t i;
+  unsigned line = 1, col = 1;
+  unsigned start_line = 1, start_col = 1;
   
-  if (!(Idx = clang_createIndex(/* excludeDeclsFromPCH */ 1,
-                                /* displayDiagnostics */ 1))) {
+  if (!(Idx = clang_createIndex(/* excludeDeclsFromPCH */ 1))) {
     fprintf(stderr, "Could not create Index\n");
     return 1;
   }
@@ -423,52 +597,34 @@ static int perform_file_scan(const char *ast_file, const char *source_file,
     return 1;
   }
   
-  line = 0;
-  prevCursor = clang_getNullCursor();
-  printed = 0;
-  start_line = last_line = 1;
-  start_col = last_col = 1;
-  
   file = clang_getFile(TU, source_file);
-  while (!feof(fp)) {
-    size_t len = 0;
-    int c;
+  for (;;) {
+    CXCursor cursor;
+    int c = fgetc(fp);
 
-    while ((c = fgetc(fp)) != EOF) {
-      len++;
-      if (c == '\n')
-        break;
+    if (c == '\n') {
+      ++line;
+      col = 1;
+    } else
+      ++col;
+
+    /* Check the cursor at this position, and dump the previous one if we have
+     * found something new.
+     */
+    cursor = clang_getCursor(TU, clang_getLocation(TU, file, line, col));
+    if ((c == EOF || !clang_equalCursors(cursor, prevCursor)) &&
+        prevCursor.kind != CXCursor_InvalidFile) {
+      print_cursor_file_scan(prevCursor, start_line, start_col,
+                             line, col, prefix);
+      start_line = line;
+      start_col = col;
     }
+    if (c == EOF)
+      break;
 
-    ++line;
-    
-    for (i = 0; i < len ; ++i) {
-      CXCursor cursor;
-      cursor = clang_getCursor(TU, clang_getLocation(TU, file, line, i+1));
-
-      if (!clang_equalCursors(cursor, prevCursor) &&
-          prevCursor.kind != CXCursor_InvalidFile) {
-        print_cursor_file_scan(prevCursor, start_line, start_col,
-                               last_line, last_col, prefix);
-        printed = 1;
-        start_line = line;
-        start_col = (unsigned) i+1;
-      }
-      else {
-        printed = 0;
-      }
-      
-      prevCursor = cursor;
-      last_line = line;
-      last_col = (unsigned) i+1;
-    }    
+    prevCursor = cursor;
   }
   
-  if (!printed && prevCursor.kind != CXCursor_InvalidFile) {
-    print_cursor_file_scan(prevCursor, start_line, start_col,
-                           last_line, last_col, prefix);
-  }  
-  
   fclose(fp);
   return 0;
 }
@@ -481,42 +637,62 @@ static int perform_file_scan(const char *ast_file, const char *source_file,
    on failure. If successful, the pointer *filename will contain newly-allocated
    memory (that will be owned by the caller) to store the file name. */
 int parse_file_line_column(const char *input, char **filename, unsigned *line, 
-                           unsigned *column) {
+                           unsigned *column, unsigned *second_line,
+                           unsigned *second_column) {
   /* Find the second colon. */
-  const char *second_colon = strrchr(input, ':'), *first_colon;
+  const char *last_colon = strrchr(input, ':');
+  unsigned values[4], i;
+  unsigned num_values = (second_line && second_column)? 4 : 2;
+
   char *endptr = 0;
-  if (!second_colon || second_colon == input) {
-    fprintf(stderr, "could not parse filename:line:column in '%s'\n", input);
+  if (!last_colon || last_colon == input) {
+    if (num_values == 4)
+      fprintf(stderr, "could not parse filename:line:column:line:column in "
+              "'%s'\n", input);
+    else
+      fprintf(stderr, "could not parse filename:line:column in '%s'\n", input);
     return 1;
   }
 
-  /* Parse the column number. */
-  *column = strtol(second_colon + 1, &endptr, 10);
-  if (*endptr != 0) {
-    fprintf(stderr, "could not parse column in '%s'\n", input);
-    return 1;
-  }
+  for (i = 0; i != num_values; ++i) {
+    const char *prev_colon;
 
-  /* Find the first colon. */
-  first_colon = second_colon - 1;
-  while (first_colon != input && *first_colon != ':')
-    --first_colon;
-  if (first_colon == input) {
-    fprintf(stderr, "could not parse line in '%s'\n", input);
-    return 1;    
-  }
+    /* Parse the next line or column. */
+    values[num_values - i - 1] = strtol(last_colon + 1, &endptr, 10);
+    if (*endptr != 0 && *endptr != ':') {
+      fprintf(stderr, "could not parse %s in '%s'\n", 
+              (i % 2 ? "column" : "line"), input);
+      return 1;
+    }
+    
+    if (i + 1 == num_values)
+      break;
 
-  /* Parse the line number. */
-  *line = strtol(first_colon + 1, &endptr, 10);
-  if (*endptr != ':') {
-    fprintf(stderr, "could not parse line in '%s'\n", input);
-    return 1;
+    /* Find the previous colon. */
+    prev_colon = last_colon - 1;
+    while (prev_colon != input && *prev_colon != ':')
+      --prev_colon;
+    if (prev_colon == input) {
+      fprintf(stderr, "could not parse %s in '%s'\n", 
+              (i % 2 == 0? "column" : "line"), input);
+      return 1;    
+    }
+
+    last_colon = prev_colon;
   }
+
+  *line = values[0];
+  *column = values[1];
   
+  if (second_line && second_column) {
+    *second_line = values[2];
+    *second_column = values[3];
+  }
+
   /* Copy the file name. */
-  *filename = (char*)malloc(first_colon - input + 1);
-  memcpy(*filename, input, first_colon - input);
-  (*filename)[first_colon - input] = 0;
+  *filename = (char*)malloc(last_colon - input + 1);
+  memcpy(*filename, input, last_colon - input);
+  (*filename)[last_colon - input] = 0;
   return 0;
 }
 
@@ -595,18 +771,21 @@ int perform_code_completion(int argc, const char **argv) {
   CXCodeCompleteResults *results = 0;
 
   input += strlen("-code-completion-at=");
-  if ((errorCode = parse_file_line_column(input, &filename, &line, &column)))
+  if ((errorCode = parse_file_line_column(input, &filename, &line, &column, 
+                                          0, 0)))
     return errorCode;
 
   if (parse_remapped_files(argc, argv, 2, &unsaved_files, &num_unsaved_files))
     return -1;
 
-  CIdx = clang_createIndex(0, 0);
+  CIdx = clang_createIndex(0);
   results = clang_codeComplete(CIdx, 
                                argv[argc - 1], argc - num_unsaved_files - 3, 
                                argv + num_unsaved_files + 2, 
                                num_unsaved_files, unsaved_files,
-                               filename, line, column);
+                               filename, line, column,
+                               PrintDiagnosticCallback, stderr);
+
   if (results) {
     unsigned i, n = results->NumResults;
     for (i = 0; i != n; ++i)
@@ -650,7 +829,7 @@ int inspect_cursor_at(int argc, const char **argv) {
     const char *input = argv[Loc + 1] + strlen("-cursor-at=");
     if ((errorCode = parse_file_line_column(input, &Locations[Loc].filename, 
                                             &Locations[Loc].line, 
-                                            &Locations[Loc].column)))
+                                            &Locations[Loc].column, 0, 0)))
       return errorCode;
   }
   
@@ -658,12 +837,14 @@ int inspect_cursor_at(int argc, const char **argv) {
                            &num_unsaved_files))
     return -1;
   
-  CIdx = clang_createIndex(0, 1);
+  CIdx = clang_createIndex(0);
   TU = clang_createTranslationUnitFromSourceFile(CIdx, argv[argc - 1],
                                   argc - num_unsaved_files - 2 - NumLocations,
                                    argv + num_unsaved_files + 1 + NumLocations,
                                                  num_unsaved_files,
-                                                 unsaved_files);
+                                                 unsaved_files,
+                                                 PrintDiagnosticCallback,
+                                                 stderr);
   if (!TU) {
     fprintf(stderr, "unable to parse input\n");
     return -1;
@@ -689,6 +870,111 @@ int inspect_cursor_at(int argc, const char **argv) {
   return 0;
 }
 
+int perform_token_annotation(int argc, const char **argv) {
+  const char *input = argv[1];
+  char *filename = 0;
+  unsigned line, second_line;
+  unsigned column, second_column;
+  CXIndex CIdx;
+  CXTranslationUnit TU = 0;
+  int errorCode;
+  struct CXUnsavedFile *unsaved_files = 0;
+  int num_unsaved_files = 0;
+  CXToken *tokens;
+  unsigned num_tokens;
+  CXSourceRange range;
+  CXSourceLocation startLoc, endLoc;
+  CXFile file = 0;
+  CXCursor *cursors = 0;
+  unsigned i;
+
+  input += strlen("-test-annotate-tokens=");
+  if ((errorCode = parse_file_line_column(input, &filename, &line, &column,
+                                          &second_line, &second_column)))
+    return errorCode;
+
+  if (parse_remapped_files(argc, argv, 2, &unsaved_files, &num_unsaved_files))
+    return -1;
+
+  CIdx = clang_createIndex(0);
+  TU = clang_createTranslationUnitFromSourceFile(CIdx, argv[argc - 1],
+                                                 argc - num_unsaved_files - 3,
+                                                 argv + num_unsaved_files + 2,
+                                                 num_unsaved_files,
+                                                 unsaved_files,
+                                                 PrintDiagnosticCallback,
+                                                 stderr);
+  if (!TU) {
+    fprintf(stderr, "unable to parse input\n");
+    clang_disposeIndex(CIdx);
+    free(filename);
+    free_remapped_files(unsaved_files, num_unsaved_files);
+    return -1;
+  }  
+  errorCode = 0;
+
+  file = clang_getFile(TU, filename);
+  if (!file) {
+    fprintf(stderr, "file %s is not in this translation unit\n", filename);
+    errorCode = -1;
+    goto teardown;
+  }
+
+  startLoc = clang_getLocation(TU, file, line, column);
+  if (clang_equalLocations(clang_getNullLocation(), startLoc)) {
+    fprintf(stderr, "invalid source location %s:%d:%d\n", filename, line, 
+            column);
+    errorCode = -1;
+    goto teardown;    
+  }
+
+  endLoc = clang_getLocation(TU, file, second_line, second_column);
+  if (clang_equalLocations(clang_getNullLocation(), endLoc)) {
+    fprintf(stderr, "invalid source location %s:%d:%d\n", filename, 
+            second_line, second_column);
+    errorCode = -1;
+    goto teardown;    
+  }
+
+  range = clang_getRange(startLoc, endLoc);
+  clang_tokenize(TU, range, &tokens, &num_tokens);
+  cursors = (CXCursor *)malloc(num_tokens * sizeof(CXCursor));
+  clang_annotateTokens(TU, tokens, num_tokens, cursors);
+  for (i = 0; i != num_tokens; ++i) {
+    const char *kind = "<unknown>";
+    CXString spelling = clang_getTokenSpelling(TU, tokens[i]);
+    CXSourceRange extent = clang_getTokenExtent(TU, tokens[i]);
+    unsigned start_line, start_column, end_line, end_column;
+
+    switch (clang_getTokenKind(tokens[i])) {
+    case CXToken_Punctuation: kind = "Punctuation"; break;
+    case CXToken_Keyword: kind = "Keyword"; break;
+    case CXToken_Identifier: kind = "Identifier"; break;
+    case CXToken_Literal: kind = "Literal"; break;
+    case CXToken_Comment: kind = "Comment"; break;
+    }
+    clang_getInstantiationLocation(clang_getRangeStart(extent), 
+                                   0, &start_line, &start_column, 0);
+    clang_getInstantiationLocation(clang_getRangeEnd(extent),
+                                   0, &end_line, &end_column, 0);
+    printf("%s: \"%s\" ", kind, clang_getCString(spelling));
+    PrintExtent(stdout, start_line, start_column, end_line, end_column);
+    if (!clang_isInvalid(cursors[i].kind)) {
+      printf(" ");
+      PrintCursor(cursors[i]);
+    }
+    printf("\n");
+  }
+  free(cursors);
+
+ teardown:
+  clang_disposeTranslationUnit(TU);
+  clang_disposeIndex(CIdx);
+  free(filename);
+  free_remapped_files(unsaved_files, num_unsaved_files);
+  return errorCode;
+}
+
 /******************************************************************************/
 /* Command line processing.                                                   */
 /******************************************************************************/
@@ -712,8 +998,11 @@ static void print_usage(void) {
     "       c-index-test -test-load-tu-usrs <AST file> <symbol filter> "
            "[FileCheck prefix]\n"
     "       c-index-test -test-load-source <symbol filter> {<args>}*\n"
-    "       c-index-test -test-load-source-usrs <symbol filter> {<args>}*\n\n");
+    "       c-index-test -test-load-source-usrs <symbol filter> {<args>}*\n");
   fprintf(stderr,
+    "       c-index-test -test-annotate-tokens=<range> {<args>}*\n"
+    "       c-index-test -test-inclusion-stack-source {<args>}*\n"
+    "       c-index-test -test-inclusion-stack-tu <AST file>\n\n"
     " <symbol filter> values:\n%s",
     "   all - load all symbols, including those from PCH\n"
     "   local - load all symbols except those in PCH\n"
@@ -733,17 +1022,26 @@ int main(int argc, const char **argv) {
   else if (argc >= 4 && strncmp(argv[1], "-test-load-tu", 13) == 0) {
     CXCursorVisitor I = GetVisitor(argv[1] + 13);
     if (I)
-      return perform_test_load_tu(argv[2], argv[3], argc >= 5 ? argv[4] : 0, I);
+      return perform_test_load_tu(argv[2], argv[3], argc >= 5 ? argv[4] : 0, I,
+                                  NULL);
   }
   else if (argc >= 4 && strncmp(argv[1], "-test-load-source", 17) == 0) {
     CXCursorVisitor I = GetVisitor(argv[1] + 17);
     if (I)
-      return perform_test_load_source(argc - 3, argv + 3, argv[2], I);
+      return perform_test_load_source(argc - 3, argv + 3, argv[2], I, NULL);
   }
   else if (argc >= 4 && strcmp(argv[1], "-test-file-scan") == 0)
     return perform_file_scan(argv[2], argv[3],
                              argc >= 5 ? argv[4] : 0);
-
+  else if (argc > 2 && strstr(argv[1], "-test-annotate-tokens=") == argv[1])
+    return perform_token_annotation(argc, argv);
+  else if (argc > 2 && strcmp(argv[1], "-test-inclusion-stack-source") == 0)
+    return perform_test_load_source(argc - 2, argv + 2, "all", NULL,
+                                    PrintInclusionStack);
+  else if (argc > 2 && strcmp(argv[1], "-test-inclusion-stack-tu") == 0)
+    return perform_test_load_tu(argv[2], "all", NULL, NULL,
+                                PrintInclusionStack);
+    
   print_usage();
   return 1;
 }