Bug 1413738 - Mozsearch clang plugin (r=mystor)
authorBill McCloskey <billm@mozilla.com>
Thu, 09 Nov 2017 15:04:33 -0800
changeset 392280 abe5e28c49908b8a94196b73a787656a546bac9c
parent 392279 0534f8329ceef79460b6c81c204155cfdf0c8460
child 392281 ec0702d55b012ccb507686d5a644e7c568524f0e
push id32916
push useraciure@mozilla.com
push dateFri, 17 Nov 2017 09:59:52 +0000
treeherdermozilla-central@a77c628829b3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmystor
bugs1413738
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1413738 - Mozsearch clang plugin (r=mystor) MozReview-Commit-ID: 53zUQ0vDMpd
build/autoconf/clang-plugin.m4
build/clang-plugin/moz.build
build/clang-plugin/mozsearch-plugin/FileOperations.cpp
build/clang-plugin/mozsearch-plugin/FileOperations.h
build/clang-plugin/mozsearch-plugin/JSONFormatter.cpp
build/clang-plugin/mozsearch-plugin/JSONFormatter.h
build/clang-plugin/mozsearch-plugin/MozsearchIndexer.cpp
build/clang-plugin/mozsearch-plugin/README
build/clang-plugin/mozsearch-plugin/StringOperations.cpp
build/clang-plugin/mozsearch-plugin/StringOperations.h
build/moz.configure/toolchain.configure
toolkit/mozapps/installer/package-name.mk
toolkit/mozapps/installer/packager.mk
toolkit/mozapps/installer/upload-files.mk
--- a/build/autoconf/clang-plugin.m4
+++ b/build/autoconf/clang-plugin.m4
@@ -151,16 +151,36 @@ if test -n "$ENABLE_CLANG_PLUGIN"; then
       LLVM_CXXFLAGS="$LLVM_CXXFLAGS -DHAS_ACCEPTS_IGNORINGPARENIMPCASTS"
     fi
 
     CLANG_PLUGIN_FLAGS="-Xclang -load -Xclang $CLANG_PLUGIN -Xclang -add-plugin -Xclang moz-check"
 
     AC_DEFINE(MOZ_CLANG_PLUGIN)
 fi
 
+if test -n "$ENABLE_MOZSEARCH_PLUGIN"; then
+    if test -z "${ENABLE_CLANG_PLUGIN}"; then
+        AC_MSG_ERROR([Can't use mozsearch plugin without --enable-clang-plugin.])
+    fi
+
+    dnl We use this construct rather than $_objdir to avoid getting /js/src in the
+    dnl path when compiling JS code.
+    OBJDIR="$(dirname $(dirname $(dirname $CLANG_PLUGIN)))"
+
+    CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -add-plugin -Xclang mozsearch-index"
+
+    dnl Parameters are: srcdir, outdir (path where output JSON is stored), objdir.
+    CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -plugin-arg-mozsearch-index -Xclang $_topsrcdir"
+    CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -plugin-arg-mozsearch-index -Xclang $OBJDIR/mozsearch_index"
+    CLANG_PLUGIN_FLAGS="$CLANG_PLUGIN_FLAGS -Xclang -plugin-arg-mozsearch-index -Xclang $OBJDIR"
+
+    AC_DEFINE(MOZ_MOZSEARCH_PLUGIN)
+fi
+
 AC_SUBST_LIST(CLANG_PLUGIN_FLAGS)
 AC_SUBST_LIST(LLVM_CXXFLAGS)
 AC_SUBST_LIST(LLVM_LDFLAGS)
 AC_SUBST_LIST(CLANG_LDFLAGS)
 
 AC_SUBST(ENABLE_CLANG_PLUGIN)
+AC_SUBST(ENABLE_MOZSEARCH_PLUGIN)
 
 ])
--- a/build/clang-plugin/moz.build
+++ b/build/clang-plugin/moz.build
@@ -36,16 +36,24 @@ UNIFIED_SOURCES += [
     'RefCountedCopyConstructorChecker.cpp',
     'RefCountedInsideLambdaChecker.cpp',
     'ScopeChecker.cpp',
     'SprintfLiteralChecker.cpp',
     'TrivialCtorDtorChecker.cpp',
     'VariableUsageHelpers.cpp',
 ]
 
+if CONFIG['ENABLE_MOZSEARCH_PLUGIN']:
+    UNIFIED_SOURCES += [
+        'mozsearch-plugin/FileOperations.cpp',
+        'mozsearch-plugin/JSONFormatter.cpp',
+        'mozsearch-plugin/MozsearchIndexer.cpp',
+        'mozsearch-plugin/StringOperations.cpp',
+    ]
+
 GENERATED_FILES += ['ThirdPartyPaths.cpp']
 third_party_paths = GENERATED_FILES['ThirdPartyPaths.cpp']
 third_party_paths.script = "ThirdPartyPaths.py:generate"
 third_party_paths.inputs = [
     '/tools/rewriting/ThirdPartyPaths.txt',
 ]
 
 DisableStlWrapping()
new file mode 100644
--- /dev/null
+++ b/build/clang-plugin/mozsearch-plugin/FileOperations.cpp
@@ -0,0 +1,129 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "FileOperations.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+#include <direct.h>
+#include <io.h>
+#include <windows.h>
+#else
+#include <sys/file.h>
+#include <sys/time.h>
+#include <unistd.h>
+#endif
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+// Make sure that all directories on path exist, excluding the final element of
+// the path.
+void ensurePath(std::string Path) {
+  size_t Pos = 0;
+  if (Path[0] == '/') {
+    Pos++;
+  }
+
+  while ((Pos = Path.find('/', Pos)) != std::string::npos) {
+    std::string Portion = Path.substr(0, Pos);
+    if (!Portion.empty()) {
+#if defined(_WIN32) || defined(_WIN64)
+      int Err = _mkdir(Portion.c_str());
+#else
+      int Err = mkdir(Portion.c_str(), 0775);
+#endif
+      if (Err == -1 && errno != EEXIST) {
+        perror("mkdir failed");
+        exit(1);
+      }
+    }
+
+    Pos++;
+  }
+}
+
+#if defined(_WIN32) || defined(_WIN64)
+AutoLockFile::AutoLockFile(const std::string &Filename) {
+  std::string Hash = Hash(filename);
+  std::string MutexName = std::string("Local\\searchfox-") + Hash;
+  std::wstring WideMutexName;
+  WideMutexName.assign(MutexName.begin(), MutexName.end());
+  Handle = CreateMutex(nullptr, false, WideMutexName.c_str());
+  if (Handle == NULL) {
+    return;
+  }
+
+  WaitForSingleObject(Handle, INFINITE);
+
+  FileDescriptor = _open(Filename.c_str(), _O_RDWR | _O_CREAT, 0666);
+}
+
+AutoLockFile::~AutoLockFile() {
+  _close(FileDescriptor);
+
+  ReleaseMutex(Handle);
+  CloseHandle(Handle);
+}
+
+bool AutoLockFile::success() {
+  return Handle != NULL && FileDescriptor != -1;
+}
+
+FILE *AutoLockFile::openFile(const char *Mode) {
+  _lseek(FileDescriptor, 0, SEEK_SET);
+  return _fdopen(_dup(FileDescriptor), Mode);
+}
+
+bool AutoLockFile::truncateFile(size_t Length) {
+  return _chsize(FileDescriptor, Length) == 0;
+}
+
+std::string getAbsolutePath(const std::string &Filename) {
+  char full[_MAX_PATH];
+  if (!_fullpath(Full, Filename.c_str(), _MAX_PATH)) {
+    return std::string("");
+  }
+  return std::string(Full);
+}
+#else
+AutoLockFile::AutoLockFile(const std::string &Filename) {
+  FileDescriptor = open(Filename.c_str(), O_RDWR | O_CREAT, 0666);
+  if (FileDescriptor == -1) {
+    return;
+  }
+
+  do {
+    int rv = flock(FileDescriptor, LOCK_EX);
+    if (rv == 0) {
+      break;
+    }
+  } while (true);
+}
+
+AutoLockFile::~AutoLockFile() { close(FileDescriptor); }
+
+bool AutoLockFile::success() { return FileDescriptor != -1; }
+
+FILE *AutoLockFile::openFile(const char *Mode) {
+  lseek(FileDescriptor, 0, SEEK_SET);
+  return fdopen(dup(FileDescriptor), Mode);
+}
+
+bool AutoLockFile::truncateFile(size_t Length) {
+  return ftruncate(FileDescriptor, Length) == 0;
+}
+
+std::string getAbsolutePath(const std::string &Filename) {
+  char Full[4096];
+  if (!realpath(Filename.c_str(), Full)) {
+    return std::string("");
+  }
+  return std::string(Full);
+}
+#endif
new file mode 100644
--- /dev/null
+++ b/build/clang-plugin/mozsearch-plugin/FileOperations.h
@@ -0,0 +1,41 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef FileOperations_h
+#define FileOperations_h
+
+#include <stdio.h>
+#include <string>
+
+#if defined(_WIN32) || defined(_WIN64)
+#include <windows.h>
+#endif
+
+// Make sure that all directories on path exist, excluding the final element of
+// the path.
+void ensurePath(std::string Path);
+
+std::string getAbsolutePath(const std::string &Filename);
+
+// Lock the given filename so that it cannot be opened by anyone else until this
+// object goes out of scope. On Windows, we use a named mutex. On POSIX
+// platforms, we use flock.
+struct AutoLockFile {
+  int FileDescriptor = -1;
+
+#if defined(_WIN32) || defined(_WIN64)
+  HANDLE Handle = NULL;
+#endif
+
+  AutoLockFile(const std::string &Filename);
+  ~AutoLockFile();
+
+  bool success();
+
+  FILE *openFile(const char *Mode);
+  bool truncateFile(size_t Length);
+};
+
+#endif
new file mode 100644
--- /dev/null
+++ b/build/clang-plugin/mozsearch-plugin/JSONFormatter.cpp
@@ -0,0 +1,119 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "JSONFormatter.h"
+
+#include <algorithm>
+#include <cassert>
+#include <string.h>
+
+static std::string replaceAll(std::string Mangled, std::string Pattern,
+                              std::string Replacement) {
+  size_t Pos = 0;
+  while ((Pos = Mangled.find(Pattern, Pos)) != std::string::npos) {
+    Mangled = Mangled.replace(Pos, Pattern.length(), Replacement);
+    Pos += Replacement.length();
+  }
+  return Mangled;
+}
+
+/**
+ * Hacky escaping logic with the goal of not upsetting the much more thorough
+ * rust JSON parsing library that actually understands UTF-8.  Double-quote
+ * and (escaping) backslash are escaped, as is tab (\t), with newlines (\r\n
+ * and \n) normalized to escaped \n.
+ *
+ * Additionally, everything that's not printable ASCII is simply erased.  The
+ * motivating file is media/openmax_il/il112/OMX_Other.h#93 which has a
+ * corrupted apostrophe as <92> in there.  The better course of action would
+ * be a validating UTF-8 parse that discards corrupt/non-printable characters.
+ * Since this is motivated by a commenting proof-of-concept and builds are
+ * already slow, I'm punting on that.
+ */
+std::string JSONFormatter::escape(std::string Input) {
+  bool NeedsEscape = false;
+  for (char C : Input) {
+    if (C == '\\' || C == '"' || C < 32 || C > 126) {
+      NeedsEscape = true;
+      break;
+    }
+  }
+
+  if (!NeedsEscape) {
+    return Input;
+  }
+
+  std::string Cur = Input;
+  Cur = replaceAll(Cur, "\\", "\\\\");
+  Cur = replaceAll(Cur, "\"", "\\\"");
+  Cur = replaceAll(Cur, "\t", "\\t");
+  Cur = replaceAll(Cur, "\r\n", "\\n");
+  Cur = replaceAll(Cur, "\n", "\\n");
+  Cur.erase(std::remove_if(Cur.begin(), Cur.end(),
+                           [](char C){ return C < 32 || C > 126; }),
+            Cur.end());
+  return Cur;
+}
+
+void JSONFormatter::add(const char *Name, const char *Value) {
+  assert(PropertyCount < kMaxProperties);
+  Properties[PropertyCount] = Property(Name, std::string(Value));
+  PropertyCount++;
+
+  // `"Name":"Value",`
+  Length += strlen(Name) + 3 + strlen(Value) + 2 + 1;
+}
+
+void JSONFormatter::add(const char *Name, std::string Value) {
+  std::string Escaped = escape(std::move(Value));
+
+  // `"Name":"Escaped",`
+  Length += strlen(Name) + 3 + Escaped.length() + 2 + 1;
+
+  assert(PropertyCount < kMaxProperties);
+  Properties[PropertyCount] = Property(Name, std::move(Escaped));
+  PropertyCount++;
+}
+
+void JSONFormatter::add(const char *Name, int Value) {
+  // 1 digit
+  assert(Value >= 0 && Value < 10);
+
+  assert(PropertyCount < kMaxProperties);
+  Properties[PropertyCount] = Property(Name, Value);
+  PropertyCount++;
+
+  // `"Name":V,`
+  Length += strlen(Name) + 3 + 2;
+}
+
+void JSONFormatter::format(std::string &Result) {
+  Result.reserve(Length + 2);
+
+  Result.push_back('{');
+  for (int I = 0; I < PropertyCount; I++) {
+    Result.push_back('"');
+    Result.append(Properties[I].Name);
+    Result.push_back('"');
+    Result.push_back(':');
+
+    if (Properties[I].IsString) {
+      Result.push_back('"');
+      Result.append(Properties[I].StringValue);
+      Result.push_back('"');
+    } else {
+      Result.push_back(Properties[I].IntValue + '0');
+    }
+
+    if (I + 1 != PropertyCount) {
+      Result.push_back(',');
+    }
+  }
+
+  Result.push_back('}');
+  Result.push_back('\n');
+
+  assert(Result.length() == Length + 2);
+}
new file mode 100644
--- /dev/null
+++ b/build/clang-plugin/mozsearch-plugin/JSONFormatter.h
@@ -0,0 +1,53 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef JSONFormatter_h
+#define JSONFormatter_h
+
+#include <memory>
+#include <string>
+
+// A very basic JSON formatter that records key/value pairs and outputs a JSON
+// object that contains only non-object data.
+class JSONFormatter {
+  // Of these fields, only mEscapedStringValue is owned by this class. All the
+  // others are expected to outlive the class (which is typically allocated
+  // on-stack).
+  struct Property {
+    const char *Name;
+    std::string StringValue;
+    int IntValue;
+    bool IsString;
+
+    Property() {}
+
+    Property(const char* Name, std::string String)
+      : Name(Name), StringValue(std::move(String)), IsString(true) {}
+
+    Property(const char* Name, int Int)
+      : Name(Name), IntValue(Int), IsString(false) {}
+  };
+
+  static const int kMaxProperties = 32;
+
+  Property Properties[kMaxProperties];
+  int PropertyCount;
+
+  // Length of the generated JSON output.
+  size_t Length;
+
+  std::string escape(std::string Input);
+
+public:
+  JSONFormatter() : PropertyCount(0), Length(0) {}
+
+  void add(const char *Name, const char *Value);
+  void add(const char *Name, std::string Value);
+  void add(const char *Name, int Value);
+
+  void format(std::string &Result);
+};
+
+#endif
new file mode 100644
--- /dev/null
+++ b/build/clang-plugin/mozsearch-plugin/MozsearchIndexer.cpp
@@ -0,0 +1,1492 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "clang/AST/AST.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/Mangle.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/Version.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendPluginRegistry.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <iostream>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <tuple>
+#include <unordered_set>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "FileOperations.h"
+#include "JSONFormatter.h"
+#include "StringOperations.h"
+
+using namespace clang;
+
+const std::string GENERATED("__GENERATED__/");
+
+// Absolute path to directory containing source code.
+std::string Srcdir;
+
+// Absolute path to objdir (including generated code).
+std::string Objdir;
+
+// Absolute path where analysis JSON output will be stored.
+std::string Outdir;
+
+#if !defined(_WIN32) && !defined(_WIN64)
+#include <sys/time.h>
+
+static double time() {
+  struct timeval Tv;
+  gettimeofday(&Tv, nullptr);
+  return double(Tv.tv_sec) + double(Tv.tv_usec) / 1000000.;
+}
+#endif
+
+// Return true if |input| is a valid C++ identifier. We don't want to generate
+// analysis information for operators, string literals, etc. by accident since
+// it trips up consumers of the data.
+static bool isValidIdentifier(std::string Input) {
+  for (char C : Input) {
+    if (!(isalpha(C) || isdigit(C) || C == '_')) {
+      return false;
+    }
+  }
+  return true;
+}
+
+class IndexConsumer;
+
+// For each C++ file seen by the analysis (.cpp or .h), we track a
+// FileInfo. This object tracks whether the file is "interesting" (i.e., whether
+// it's in the source dir or the objdir). We also store the analysis output
+// here.
+struct FileInfo {
+  FileInfo(std::string &Rname) : Realname(Rname) {
+    if (Rname.compare(0, Objdir.length(), Objdir) == 0) {
+      // We're in the objdir, so we are probably a generated header
+      // We use the escape character to indicate the objdir nature.
+      // Note that output also has the `/' already placed
+      Interesting = true;
+      Realname.replace(0, Objdir.length(), GENERATED);
+      return;
+    }
+
+    Interesting = Rname.compare(0, Srcdir.length(), Srcdir) == 0;
+    if (Interesting) {
+      // Remove the trailing `/' as well.
+      Realname.erase(0, Srcdir.length() + 1);
+    }
+  }
+  std::string Realname;
+  std::vector<std::string> Output;
+  bool Interesting;
+};
+
+class IndexConsumer;
+
+class PreprocessorHook : public PPCallbacks {
+  IndexConsumer *Indexer;
+
+public:
+  PreprocessorHook(IndexConsumer *C) : Indexer(C) {}
+
+  virtual void MacroDefined(const Token &Tok,
+                            const MacroDirective *Md) override;
+
+  virtual void MacroExpands(const Token &Tok, const MacroDefinition &Md,
+                            SourceRange Range, const MacroArgs *Ma) override;
+#if CLANG_VERSION_MAJOR >= 5
+  virtual void MacroUndefined(const Token &tok, const MacroDefinition &md,
+                              const MacroDirective *Undef) override;
+#else
+  virtual void MacroUndefined(const Token &Tok,
+                              const MacroDefinition &Md) override;
+#endif
+  virtual void Defined(const Token &Tok, const MacroDefinition &Md,
+                       SourceRange Range) override;
+  virtual void Ifdef(SourceLocation Loc, const Token &Tok,
+                     const MacroDefinition &Md) override;
+  virtual void Ifndef(SourceLocation Loc, const Token &Tok,
+                      const MacroDefinition &Md) override;
+};
+
+class IndexConsumer : public ASTConsumer,
+                      public RecursiveASTVisitor<IndexConsumer>,
+                      public DiagnosticConsumer {
+private:
+  CompilerInstance &CI;
+  SourceManager &SM;
+  std::map<FileID, std::unique_ptr<FileInfo>> FileMap;
+  MangleContext *CurMangleContext;
+  ASTContext *AstContext;
+
+  typedef RecursiveASTVisitor<IndexConsumer> Super;
+
+  // Tracks the set of declarations that the current expression/statement is
+  // nested inside of.
+  struct AutoSetContext {
+    AutoSetContext(IndexConsumer *Self, NamedDecl *Context)
+        : Self(Self), Prev(Self->CurDeclContext), Decl(Context) {
+      Self->CurDeclContext = this;
+    }
+
+    ~AutoSetContext() { Self->CurDeclContext = Prev; }
+
+    IndexConsumer *Self;
+    AutoSetContext *Prev;
+    NamedDecl *Decl;
+  };
+  AutoSetContext *CurDeclContext;
+
+  FileInfo *getFileInfo(SourceLocation Loc) {
+    FileID Id = SM.getFileID(Loc);
+
+    std::map<FileID, std::unique_ptr<FileInfo>>::iterator It;
+    It = FileMap.find(Id);
+    if (It == FileMap.end()) {
+      // We haven't seen this file before. We need to make the FileInfo
+      // structure information ourselves
+      std::string Filename = SM.getFilename(Loc);
+      std::string Absolute = getAbsolutePath(Filename);
+      if (Absolute.empty()) {
+        Absolute = Filename;
+      }
+      std::unique_ptr<FileInfo> Info = llvm::make_unique<FileInfo>(Absolute);
+      It = FileMap.insert(std::make_pair(Id, std::move(Info))).first;
+    }
+    return It->second.get();
+  }
+
+  // Helpers for processing declarations
+  // Should we ignore this location?
+  bool isInterestingLocation(SourceLocation Loc) {
+    if (Loc.isInvalid()) {
+      return false;
+    }
+
+    return getFileInfo(Loc)->Interesting;
+  }
+
+  std::string locationToString(SourceLocation Loc, size_t Length = 0) {
+    std::pair<FileID, unsigned> Pair = SM.getDecomposedLoc(Loc);
+
+    bool IsInvalid;
+    unsigned Line = SM.getLineNumber(Pair.first, Pair.second, &IsInvalid);
+    if (IsInvalid) {
+      return "";
+    }
+    unsigned Column = SM.getColumnNumber(Pair.first, Pair.second, &IsInvalid);
+    if (IsInvalid) {
+      return "";
+    }
+
+    if (Length) {
+      return stringFormat("%05d:%d-%d", Line, Column - 1, Column - 1 + Length);
+    } else {
+      return stringFormat("%05d:%d", Line, Column - 1);
+    }
+  }
+
+  std::string lineRangeToString(SourceRange Range) {
+    std::pair<FileID, unsigned> Begin = SM.getDecomposedLoc(Range.getBegin());
+    std::pair<FileID, unsigned> End = SM.getDecomposedLoc(Range.getEnd());
+
+    bool IsInvalid;
+    unsigned Line1 = SM.getLineNumber(Begin.first, Begin.second, &IsInvalid);
+    if (IsInvalid) {
+      return "";
+    }
+    unsigned Line2 = SM.getLineNumber(End.first, End.second, &IsInvalid);
+    if (IsInvalid) {
+      return "";
+    }
+
+    return stringFormat("%d-%d", Line1, Line2);
+  }
+
+  // Returns the qualified name of `d` without considering template parameters.
+  std::string getQualifiedName(const NamedDecl *D) {
+    const DeclContext *Ctx = D->getDeclContext();
+    if (Ctx->isFunctionOrMethod()) {
+      return D->getQualifiedNameAsString();
+    }
+
+    std::vector<const DeclContext *> Contexts;
+
+    // Collect contexts.
+    while (Ctx && isa<NamedDecl>(Ctx)) {
+      Contexts.push_back(Ctx);
+      Ctx = Ctx->getParent();
+    }
+
+    std::string Result;
+
+    std::reverse(Contexts.begin(), Contexts.end());
+
+    for (const DeclContext *DC : Contexts) {
+      if (const auto *Spec = dyn_cast<ClassTemplateSpecializationDecl>(DC)) {
+        Result += Spec->getNameAsString();
+
+        if (Spec->getSpecializationKind() == TSK_ExplicitSpecialization) {
+          std::string Backing;
+          llvm::raw_string_ostream Stream(Backing);
+          const TemplateArgumentList &TemplateArgs = Spec->getTemplateArgs();
+#if CLANG_VERSION_MAJOR > 3 ||                                                 \
+    (CLANG_VERSION_MAJOR == 3 && CLANG_VERSION_MINOR >= 9)
+          TemplateSpecializationType::PrintTemplateArgumentList(
+              Stream, TemplateArgs.asArray(), PrintingPolicy(CI.getLangOpts()));
+#else
+          TemplateSpecializationType::PrintTemplateArgumentList(
+              stream, templateArgs.data(), templateArgs.size(),
+              PrintingPolicy(CI.getLangOpts()));
+#endif
+          Result += Stream.str();
+        }
+      } else if (const auto *Nd = dyn_cast<NamespaceDecl>(DC)) {
+        if (Nd->isAnonymousNamespace() || Nd->isInline()) {
+          continue;
+        }
+        Result += Nd->getNameAsString();
+      } else if (const auto *Rd = dyn_cast<RecordDecl>(DC)) {
+        if (!Rd->getIdentifier()) {
+          Result += "(anonymous)";
+        } else {
+          Result += Rd->getNameAsString();
+        }
+      } else if (const auto *Fd = dyn_cast<FunctionDecl>(DC)) {
+        Result += Fd->getNameAsString();
+      } else if (const auto *Ed = dyn_cast<EnumDecl>(DC)) {
+        // C++ [dcl.enum]p10: Each enum-name and each unscoped
+        // enumerator is declared in the scope that immediately contains
+        // the enum-specifier. Each scoped enumerator is declared in the
+        // scope of the enumeration.
+        if (Ed->isScoped() || Ed->getIdentifier())
+          Result += Ed->getNameAsString();
+        else
+          continue;
+      } else {
+        Result += cast<NamedDecl>(DC)->getNameAsString();
+      }
+      Result += "::";
+    }
+
+    if (D->getDeclName())
+      Result += D->getNameAsString();
+    else
+      Result += "(anonymous)";
+
+    return Result;
+  }
+
+  std::string mangleLocation(SourceLocation Loc,
+                             std::string Backup = std::string()) {
+    FileInfo *F = getFileInfo(Loc);
+    std::string Filename = F->Realname;
+    if (Filename.length() == 0 && Backup.length() != 0) {
+      return Backup;
+    }
+    return hash(Filename + std::string("@") + locationToString(Loc));
+  }
+
+  std::string mangleQualifiedName(std::string Name) {
+    std::replace(Name.begin(), Name.end(), ' ', '_');
+    return Name;
+  }
+
+  std::string getMangledName(clang::MangleContext *Ctx,
+                             const clang::NamedDecl *Decl) {
+    if (isa<FunctionDecl>(Decl) || isa<VarDecl>(Decl)) {
+      const DeclContext *DC = Decl->getDeclContext();
+      if (isa<TranslationUnitDecl>(DC) || isa<NamespaceDecl>(DC) ||
+          isa<LinkageSpecDecl>(DC) ||
+          // isa<ExternCContextDecl>(DC) ||
+          isa<TagDecl>(DC)) {
+        llvm::SmallVector<char, 512> Output;
+        llvm::raw_svector_ostream Out(Output);
+        if (const CXXConstructorDecl *D = dyn_cast<CXXConstructorDecl>(Decl)) {
+          Ctx->mangleCXXCtor(D, CXXCtorType::Ctor_Complete, Out);
+        } else if (const CXXDestructorDecl *D =
+                       dyn_cast<CXXDestructorDecl>(Decl)) {
+          Ctx->mangleCXXDtor(D, CXXDtorType::Dtor_Complete, Out);
+        } else {
+          Ctx->mangleName(Decl, Out);
+        }
+        return Out.str().str();
+      } else {
+        return std::string("V_") + mangleLocation(Decl->getLocation()) +
+               std::string("_") + hash(Decl->getName());
+      }
+    } else if (isa<TagDecl>(Decl) || isa<TypedefNameDecl>(Decl)) {
+      if (!Decl->getIdentifier()) {
+        // Anonymous.
+        return std::string("T_") + mangleLocation(Decl->getLocation());
+      }
+
+      return std::string("T_") + mangleQualifiedName(getQualifiedName(Decl));
+    } else if (isa<NamespaceDecl>(Decl) || isa<NamespaceAliasDecl>(Decl)) {
+      if (!Decl->getIdentifier()) {
+        // Anonymous.
+        return std::string("NS_") + mangleLocation(Decl->getLocation());
+      }
+
+      return std::string("NS_") + mangleQualifiedName(getQualifiedName(Decl));
+    } else if (const FieldDecl *D2 = dyn_cast<FieldDecl>(Decl)) {
+      const RecordDecl *Record = D2->getParent();
+      return std::string("F_<") + getMangledName(Ctx, Record) + ">_" +
+             toString(D2->getFieldIndex());
+    } else if (const EnumConstantDecl *D2 = dyn_cast<EnumConstantDecl>(Decl)) {
+      const DeclContext *DC = Decl->getDeclContext();
+      if (const NamedDecl *Named = dyn_cast<NamedDecl>(DC)) {
+        return std::string("E_<") + getMangledName(Ctx, Named) + ">_" +
+               D2->getNameAsString();
+      }
+    }
+
+    assert(false);
+    return std::string("");
+  }
+
+  void debugLocation(SourceLocation Loc) {
+    std::string S = locationToString(Loc);
+    StringRef Filename = SM.getFilename(Loc);
+    printf("--> %s %s\n", std::string(Filename).c_str(), S.c_str());
+  }
+
+  void debugRange(SourceRange Range) {
+    printf("Range\n");
+    debugLocation(Range.getBegin());
+    debugLocation(Range.getEnd());
+  }
+
+public:
+  IndexConsumer(CompilerInstance &CI)
+      : CI(CI), SM(CI.getSourceManager()), CurMangleContext(nullptr),
+        AstContext(nullptr), CurDeclContext(nullptr), TemplateStack(nullptr) {
+    CI.getPreprocessor().addPPCallbacks(
+        llvm::make_unique<PreprocessorHook>(this));
+  }
+
+  virtual DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const {
+    return new IndexConsumer(CI);
+  }
+
+#if !defined(_WIN32) && !defined(_WIN64)
+  struct AutoTime {
+    AutoTime(double *Counter) : Counter(Counter), Start(time()) {}
+    ~AutoTime() {
+      if (Start) {
+        *Counter += time() - Start;
+      }
+    }
+    void stop() {
+      *Counter += time() - Start;
+      Start = 0;
+    }
+    double *Counter;
+    double Start;
+  };
+#endif
+
+  // All we need is to follow the final declaration.
+  virtual void HandleTranslationUnit(ASTContext &Ctx) {
+    CurMangleContext =
+      clang::ItaniumMangleContext::create(Ctx, CI.getDiagnostics());
+
+    AstContext = &Ctx;
+    TraverseDecl(Ctx.getTranslationUnitDecl());
+
+    // Emit the JSON data for all files now.
+    std::map<FileID, std::unique_ptr<FileInfo>>::iterator It;
+    for (It = FileMap.begin(); It != FileMap.end(); It++) {
+      if (!It->second->Interesting) {
+        continue;
+      }
+
+      FileInfo &Info = *It->second;
+
+      std::string Filename = Outdir;
+      Filename += It->second->Realname;
+
+      ensurePath(Filename);
+
+      // We lock the output file in case some other clang process is trying to
+      // write to it at the same time.
+      AutoLockFile Lock(Filename);
+
+      if (!Lock.success()) {
+        continue;
+      }
+
+      std::vector<std::string> Lines;
+
+      // Read all the existing lines in from the output file. Rather than
+      // overwrite them, we want to merge our results with what was already
+      // there. This ensures that header files that are included multiple times
+      // in different ways are analyzed completely.
+      char Buffer[65536];
+      FILE *Fp = Lock.openFile("r");
+      while (fgets(Buffer, sizeof(Buffer), Fp)) {
+        Lines.push_back(std::string(Buffer));
+      }
+      fclose(Fp);
+
+      // Insert the newly generated analysis data into what was read. Sort the
+      // results and then remove duplicates.
+      Lines.insert(Lines.end(), Info.Output.begin(), Info.Output.end());
+      std::sort(Lines.begin(), Lines.end());
+
+      std::vector<std::string> Nodupes;
+      std::unique_copy(Lines.begin(), Lines.end(), std::back_inserter(Nodupes));
+
+      // Overwrite the output file with the merged data. Since we have the lock,
+      // this will happen atomically.
+      Fp = Lock.openFile("w");
+      size_t Length = 0;
+      for (std::string &Line : Nodupes) {
+        Length += Line.length();
+        fwrite(Line.c_str(), Line.length(), 1, Fp);
+      }
+      fclose(Fp);
+
+      if (!Lock.truncateFile(Length)) {
+        return;
+      }
+    }
+  }
+
+  // Return a list of mangled names of all the methods that the given method
+  // overrides.
+  void findOverriddenMethods(const CXXMethodDecl *Method,
+                             std::vector<std::string> &Symbols) {
+    std::string Mangled = getMangledName(CurMangleContext, Method);
+    Symbols.push_back(Mangled);
+
+    CXXMethodDecl::method_iterator Iter = Method->begin_overridden_methods();
+    CXXMethodDecl::method_iterator End = Method->end_overridden_methods();
+    for (; Iter != End; Iter++) {
+      const CXXMethodDecl *Decl = *Iter;
+      if (Decl->isTemplateInstantiation()) {
+        Decl = dyn_cast<CXXMethodDecl>(Decl->getTemplateInstantiationPattern());
+      }
+      return findOverriddenMethods(Decl, Symbols);
+    }
+  }
+
+  // Unfortunately, we have to override all these methods in order to track the
+  // context we're inside.
+
+  bool TraverseEnumDecl(EnumDecl *D) {
+    AutoSetContext Asc(this, D);
+    return Super::TraverseEnumDecl(D);
+  }
+  bool TraverseRecordDecl(RecordDecl *D) {
+    AutoSetContext Asc(this, D);
+    return Super::TraverseRecordDecl(D);
+  }
+  bool TraverseCxxRecordDecl(CXXRecordDecl *D) {
+    AutoSetContext Asc(this, D);
+    return Super::TraverseCXXRecordDecl(D);
+  }
+  bool TraverseFunctionDecl(FunctionDecl *D) {
+    AutoSetContext Asc(this, D);
+    const FunctionDecl *Def;
+    // (See the larger AutoTemplateContext comment for more information.) If a
+    // method on a templated class is declared out-of-line, we need to analyze
+    // the definition inside the scope of the template or else we won't properly
+    // handle member access on the templated type.
+    if (TemplateStack && D->isDefined(Def) && Def && D != Def) {
+      TraverseFunctionDecl(const_cast<FunctionDecl *>(Def));
+    }
+    return Super::TraverseFunctionDecl(D);
+  }
+  bool TraverseCxxMethodDecl(CXXMethodDecl *D) {
+    AutoSetContext Asc(this, D);
+    const FunctionDecl *Def;
+    // See TraverseFunctionDecl.
+    if (TemplateStack && D->isDefined(Def) && Def && D != Def) {
+      TraverseFunctionDecl(const_cast<FunctionDecl *>(Def));
+    }
+    return Super::TraverseCXXMethodDecl(D);
+  }
+  bool TraverseCxxConstructorDecl(CXXConstructorDecl *D) {
+    AutoSetContext Asc(this, D);
+    const FunctionDecl *Def;
+    // See TraverseFunctionDecl.
+    if (TemplateStack && D->isDefined(Def) && Def && D != Def) {
+      TraverseFunctionDecl(const_cast<FunctionDecl *>(Def));
+    }
+    return Super::TraverseCXXConstructorDecl(D);
+  }
+  bool TraverseCxxConversionDecl(CXXConversionDecl *D) {
+    AutoSetContext Asc(this, D);
+    const FunctionDecl *Def;
+    // See TraverseFunctionDecl.
+    if (TemplateStack && D->isDefined(Def) && Def && D != Def) {
+      TraverseFunctionDecl(const_cast<FunctionDecl *>(Def));
+    }
+    return Super::TraverseCXXConversionDecl(D);
+  }
+  bool TraverseCxxDestructorDecl(CXXDestructorDecl *D) {
+    AutoSetContext Asc(this, D);
+    const FunctionDecl *Def;
+    // See TraverseFunctionDecl.
+    if (TemplateStack && D->isDefined(Def) && Def && D != Def) {
+      TraverseFunctionDecl(const_cast<FunctionDecl *>(Def));
+    }
+    return Super::TraverseCXXDestructorDecl(D);
+  }
+
+  // Used to keep track of the context in which a token appears.
+  struct Context {
+    // Ultimately this becomes the "context" JSON property.
+    std::string Name;
+
+    // Ultimately this becomes the "contextsym" JSON property.
+    std::vector<std::string> Symbols;
+
+    Context() {}
+    Context(std::string Name, std::vector<std::string> Symbols)
+        : Name(Name), Symbols(Symbols) {}
+  };
+
+  Context translateContext(NamedDecl *D) {
+    const FunctionDecl *F = dyn_cast<FunctionDecl>(D);
+    if (F && F->isTemplateInstantiation()) {
+      D = F->getTemplateInstantiationPattern();
+    }
+
+    std::vector<std::string> Symbols = {getMangledName(CurMangleContext, D)};
+    if (CXXMethodDecl::classof(D)) {
+      Symbols.clear();
+      findOverriddenMethods(dyn_cast<CXXMethodDecl>(D), Symbols);
+    }
+    return Context(D->getQualifiedNameAsString(), Symbols);
+  }
+
+  Context getContext(SourceLocation Loc) {
+    if (SM.isMacroBodyExpansion(Loc)) {
+      // If we're inside a macro definition, we don't return any context. It
+      // will probably not be what the user expects if we do.
+      return Context();
+    }
+
+    if (CurDeclContext) {
+      return translateContext(CurDeclContext->Decl);
+    }
+    return Context();
+  }
+
+  // Similar to GetContext(SourceLocation), but it skips the declaration passed
+  // in. This is useful if we want the context of a declaration that's already
+  // on the stack.
+  Context getContext(Decl *D) {
+    if (SM.isMacroBodyExpansion(D->getLocation())) {
+      // If we're inside a macro definition, we don't return any context. It
+      // will probably not be what the user expects if we do.
+      return Context();
+    }
+
+    AutoSetContext *Ctxt = CurDeclContext;
+    while (Ctxt) {
+      if (Ctxt->Decl != D) {
+        return translateContext(Ctxt->Decl);
+      }
+      Ctxt = Ctxt->Prev;
+    }
+    return Context();
+  }
+
+  static std::string concatSymbols(const std::vector<std::string> Symbols) {
+    if (Symbols.empty()) {
+      return "";
+    }
+
+    size_t Total = 0;
+    for (auto It = Symbols.begin(); It != Symbols.end(); It++) {
+      Total += It->length();
+    }
+    Total += Symbols.size() - 1;
+
+    std::string SymbolList;
+    SymbolList.reserve(Total);
+
+    for (auto It = Symbols.begin(); It != Symbols.end(); It++) {
+      std::string Symbol = *It;
+
+      if (It != Symbols.begin()) {
+        SymbolList.push_back(',');
+      }
+      SymbolList.append(Symbol);
+    }
+
+    return SymbolList;
+  }
+
+  // Analyzing template code is tricky. Suppose we have this code:
+  //
+  //   template<class T>
+  //   bool Foo(T* ptr) { return T::StaticMethod(ptr); }
+  //
+  // If we analyze the body of Foo without knowing the type T, then we will not
+  // be able to generate any information for StaticMethod. However, analyzing
+  // Foo for every possible instantiation is inefficient and it also generates
+  // too much data in some cases. For example, the following code would generate
+  // one definition of Baz for every instantiation, which is undesirable:
+  //
+  //   template<class T>
+  //   class Bar { struct Baz { ... }; };
+  //
+  // To solve this problem, we analyze templates only once. We do so in a
+  // GatherDependent mode where we look for "dependent scoped member
+  // expressions" (i.e., things like StaticMethod). We keep track of the
+  // locations of these expressions. If we find one or more of them, we analyze
+  // the template for each instantiation, in an AnalyzeDependent mode. This mode
+  // ignores all source locations except for the ones where we found dependent
+  // scoped member expressions before. For these locations, we generate a
+  // separate JSON result for each instantiation.
+  struct AutoTemplateContext {
+    AutoTemplateContext(IndexConsumer *Self)
+        : Self(Self), CurMode(Mode::GatherDependent),
+          Parent(Self->TemplateStack) {
+      Self->TemplateStack = this;
+    }
+
+    ~AutoTemplateContext() { Self->TemplateStack = Parent; }
+
+    // We traverse templates in two modes:
+    enum class Mode {
+      // Gather mode does not traverse into specializations. It looks for
+      // locations where it would help to have more info from template
+      // specializations.
+      GatherDependent,
+
+      // Analyze mode traverses into template specializations and records
+      // information about token locations saved in gather mode.
+      AnalyzeDependent,
+    };
+
+    // We found a dependent scoped member expression! Keep track of it for
+    // later.
+    void visitDependent(SourceLocation Loc) {
+      if (CurMode == Mode::AnalyzeDependent) {
+        return;
+      }
+
+      DependentLocations.insert(Loc.getRawEncoding());
+      if (Parent) {
+        Parent->visitDependent(Loc);
+      }
+    }
+
+    // Do we need to perform the extra AnalyzeDependent passes (one per
+    // instantiation)?
+    bool needsAnalysis() const {
+      if (!DependentLocations.empty()) {
+        return true;
+      }
+      if (Parent) {
+        return Parent->needsAnalysis();
+      }
+      return false;
+    }
+
+    void switchMode() { CurMode = Mode::AnalyzeDependent; }
+
+    // Do we want to analyze each template instantiation separately?
+    bool shouldVisitTemplateInstantiations() const {
+      if (CurMode == Mode::AnalyzeDependent) {
+        return true;
+      }
+      if (Parent) {
+        return Parent->shouldVisitTemplateInstantiations();
+      }
+      return false;
+    }
+
+    // For a given expression/statement, should we emit JSON data for it?
+    bool shouldVisit(SourceLocation Loc) {
+      if (CurMode == Mode::GatherDependent) {
+        return true;
+      }
+      if (DependentLocations.find(Loc.getRawEncoding()) !=
+          DependentLocations.end()) {
+        return true;
+      }
+      if (Parent) {
+        return Parent->shouldVisit(Loc);
+      }
+      return false;
+    }
+
+  private:
+    IndexConsumer *Self;
+    Mode CurMode;
+    std::unordered_set<unsigned> DependentLocations;
+    AutoTemplateContext *Parent;
+  };
+
+  AutoTemplateContext *TemplateStack;
+
+  bool shouldVisitTemplateInstantiations() const {
+    if (TemplateStack) {
+      return TemplateStack->shouldVisitTemplateInstantiations();
+    }
+    return false;
+  }
+
+  bool TraverseClassTemplateDecl(ClassTemplateDecl *D) {
+    AutoTemplateContext Atc(this);
+    Super::TraverseClassTemplateDecl(D);
+
+    if (!Atc.needsAnalysis()) {
+      return true;
+    }
+
+    Atc.switchMode();
+
+    if (D != D->getCanonicalDecl()) {
+      return true;
+    }
+
+    for (auto *Spec : D->specializations()) {
+      for (auto *Rd : Spec->redecls()) {
+        // We don't want to visit injected-class-names in this traversal.
+        if (cast<CXXRecordDecl>(Rd)->isInjectedClassName())
+          continue;
+
+        TraverseDecl(Rd);
+      }
+    }
+
+    return true;
+  }
+
+  bool TraverseFunctionTemplateDecl(FunctionTemplateDecl *D) {
+    AutoTemplateContext Atc(this);
+    Super::TraverseFunctionTemplateDecl(D);
+
+    if (!Atc.needsAnalysis()) {
+      return true;
+    }
+
+    Atc.switchMode();
+
+    if (D != D->getCanonicalDecl()) {
+      return true;
+    }
+
+    for (auto *Spec : D->specializations()) {
+      for (auto *Rd : Spec->redecls()) {
+        TraverseDecl(Rd);
+      }
+    }
+
+    return true;
+  }
+
+  bool shouldVisit(SourceLocation Loc) {
+    if (TemplateStack) {
+      return TemplateStack->shouldVisit(Loc);
+    }
+    return true;
+  }
+
+  enum {
+    NoCrossref = 1,
+  };
+
+  // This is the only function that emits analysis JSON data. It should be
+  // called for each identifier that corresponds to a symbol.
+  void visitIdentifier(const char *Kind, const char *SyntaxKind,
+                       std::string QualName, SourceLocation Loc,
+                       const std::vector<std::string> &Symbols,
+                       Context TokenContext = Context(), int Flags = 0,
+                       SourceRange PeekRange = SourceRange()) {
+    if (!shouldVisit(Loc)) {
+      return;
+    }
+
+    // Find the file positions corresponding to the token.
+    unsigned StartOffset = SM.getFileOffset(Loc);
+    unsigned EndOffset =
+        StartOffset + Lexer::MeasureTokenLength(Loc, SM, CI.getLangOpts());
+
+    std::string LocStr = locationToString(Loc, EndOffset - StartOffset);
+    std::string RangeStr = locationToString(Loc, EndOffset - StartOffset);
+    std::string PeekRangeStr;
+
+    // Get the token's characters so we can make sure it's a valid token.
+    const char *StartChars = SM.getCharacterData(Loc);
+    std::string Text(StartChars, EndOffset - StartOffset);
+    if (!isValidIdentifier(Text)) {
+      return;
+    }
+
+    FileInfo *F = getFileInfo(Loc);
+
+    std::string SymbolList;
+
+    // Reserve space in symbolList for everything in `symbols`. `symbols` can
+    // contain some very long strings.
+    size_t Total = 0;
+    for (auto It = Symbols.begin(); It != Symbols.end(); It++) {
+      Total += It->length();
+    }
+
+    // Space for commas.
+    Total += Symbols.size() - 1;
+    SymbolList.reserve(Total);
+
+    // For each symbol, generate one "target":1 item. We want to find this line
+    // if someone searches for any one of these symbols.
+    for (auto It = Symbols.begin(); It != Symbols.end(); It++) {
+      std::string Symbol = *It;
+
+      if (!(Flags & NoCrossref)) {
+        JSONFormatter Fmt;
+
+        Fmt.add("loc", LocStr);
+        Fmt.add("target", 1);
+        Fmt.add("kind", Kind);
+        Fmt.add("pretty", QualName);
+        Fmt.add("sym", Symbol);
+        if (!TokenContext.Name.empty()) {
+          Fmt.add("context", TokenContext.Name);
+        }
+        std::string ContextSymbol = concatSymbols(TokenContext.Symbols);
+        if (!ContextSymbol.empty()) {
+          Fmt.add("contextsym", ContextSymbol);
+        }
+        if (PeekRange.isValid()) {
+          PeekRangeStr = lineRangeToString(PeekRange);
+          if (!PeekRangeStr.empty()) {
+            Fmt.add("peekRange", PeekRangeStr);
+          }
+        }
+
+        std::string S;
+        Fmt.format(S);
+        F->Output.push_back(std::move(S));
+      }
+
+      if (It != Symbols.begin()) {
+        SymbolList.push_back(',');
+      }
+      SymbolList.append(Symbol);
+    }
+
+    // Generate a single "source":1 for all the symbols. If we search from here,
+    // we want to union the results for every symbol in `symbols`.
+    JSONFormatter Fmt;
+
+    Fmt.add("loc", RangeStr);
+    Fmt.add("source", 1);
+
+    std::string Syntax;
+    if (Flags & NoCrossref) {
+      Fmt.add("syntax", "");
+    } else {
+      Syntax = Kind;
+      Syntax.push_back(',');
+      Syntax.append(SyntaxKind);
+      Fmt.add("syntax", Syntax);
+    }
+
+    std::string Pretty(SyntaxKind);
+    Pretty.push_back(' ');
+    Pretty.append(QualName);
+    Fmt.add("pretty", Pretty);
+
+    Fmt.add("sym", SymbolList);
+
+    if (Flags & NoCrossref) {
+      Fmt.add("no_crossref", 1);
+    }
+
+    std::string Buf;
+    Fmt.format(Buf);
+    F->Output.push_back(std::move(Buf));
+  }
+
+  void visitIdentifier(const char *Kind, const char *SyntaxKind,
+                       std::string QualName, SourceLocation Loc, std::string Symbol,
+                       Context TokenContext = Context(), int Flags = 0,
+                       SourceRange PeekRange = SourceRange()) {
+    std::vector<std::string> V = {Symbol};
+    visitIdentifier(Kind, SyntaxKind, QualName, Loc, V, TokenContext, Flags, PeekRange);
+  }
+
+  void normalizeLocation(SourceLocation *Loc) {
+    *Loc = SM.getSpellingLoc(*Loc);
+  }
+
+  SourceRange getFunctionPeekRange(FunctionDecl* D) {
+    // We always start at the start of the function decl, which may include the
+    // return type on a separate line.
+    SourceLocation Start = D->getLocStart();
+
+    // By default, we end at the line containing the function's name.
+    SourceLocation End = D->getLocation();
+
+    std::pair<FileID, unsigned> FuncLoc = SM.getDecomposedLoc(End);
+
+    // But if there are parameters, we want to include those as well.
+    for (ParmVarDecl* Param : D->parameters()) {
+      std::pair<FileID, unsigned> ParamLoc = SM.getDecomposedLoc(Param->getLocation());
+
+      // It's possible there are macros involved or something. We don't include
+      // the parameters in that case.
+      if (ParamLoc.first == FuncLoc.first) {
+        // Assume parameters are in order, so we always take the last one.
+        End = Param->getLocEnd();
+      }
+    }
+
+    return SourceRange(Start, End);
+  }
+
+  SourceRange getTagPeekRange(TagDecl* D) {
+    SourceLocation Start = D->getLocStart();
+
+    // By default, we end at the line containing the name.
+    SourceLocation End = D->getLocation();
+
+    std::pair<FileID, unsigned> FuncLoc = SM.getDecomposedLoc(End);
+
+    if (CXXRecordDecl* D2 = dyn_cast<CXXRecordDecl>(D)) {
+      // But if there are parameters, we want to include those as well.
+      for (CXXBaseSpecifier& Base : D2->bases()) {
+        std::pair<FileID, unsigned> Loc = SM.getDecomposedLoc(Base.getLocEnd());
+
+        // It's possible there are macros involved or something. We don't include
+        // the parameters in that case.
+        if (Loc.first == FuncLoc.first) {
+          // Assume parameters are in order, so we always take the last one.
+          End = Base.getLocEnd();
+        }
+      }
+    }
+
+    return SourceRange(Start, End);
+  }
+
+  SourceRange getCommentRange(NamedDecl* D) {
+    const RawComment* RC =
+      AstContext->getRawCommentForDeclNoCache(D);
+    if (!RC) {
+      return SourceRange();
+    }
+
+    return RC->getSourceRange();
+  }
+
+  SourceRange combineRanges(SourceRange Range1, SourceRange Range2) {
+    if (Range1.isInvalid()) {
+      return Range2;
+    }
+    if (Range2.isInvalid()) {
+      return Range1;
+    }
+
+    std::pair<FileID, unsigned> Begin1 = SM.getDecomposedLoc(Range1.getBegin());
+    std::pair<FileID, unsigned> End1 = SM.getDecomposedLoc(Range1.getEnd());
+    std::pair<FileID, unsigned> Begin2 = SM.getDecomposedLoc(Range2.getBegin());
+    std::pair<FileID, unsigned> End2 = SM.getDecomposedLoc(Range2.getEnd());
+
+    if (End1.first != Begin2.first) {
+      // Something weird is probably happening with the preprocessor. Just
+      // return the first range.
+      return Range1;
+    }
+
+    // See which range comes first.
+    if (Begin1.second <= End2.second) {
+      return SourceRange(Range1.getBegin(), Range2.getEnd());
+    } else {
+      return SourceRange(Range2.getBegin(), Range1.getEnd());
+    }
+  }
+
+  SourceRange validateRange(SourceLocation Loc, SourceRange Range) {
+    std::pair<FileID, unsigned> Decomposed = SM.getDecomposedLoc(Loc);
+    std::pair<FileID, unsigned> Begin = SM.getDecomposedLoc(Range.getBegin());
+    std::pair<FileID, unsigned> End = SM.getDecomposedLoc(Range.getEnd());
+
+    if (Begin.first != Decomposed.first || End.first != Decomposed.first) {
+      return SourceRange();
+    }
+
+    if (Begin.second >= End.second) {
+      return SourceRange();
+    }
+
+    return Range;
+  }
+
+  bool VisitNamedDecl(NamedDecl *D) {
+    SourceLocation Loc = D->getLocation();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    if (isa<ParmVarDecl>(D) && !D->getDeclName().getAsIdentifierInfo()) {
+      // Unnamed parameter in function proto.
+      return true;
+    }
+
+    int Flags = 0;
+    const char *Kind = "def";
+    const char *PrettyKind = "?";
+    SourceRange PeekRange(D->getLocStart(), D->getLocEnd());
+    if (FunctionDecl *D2 = dyn_cast<FunctionDecl>(D)) {
+      if (D2->isTemplateInstantiation()) {
+        D = D2->getTemplateInstantiationPattern();
+      }
+      Kind = D2->isThisDeclarationADefinition() ? "def" : "decl";
+      PrettyKind = "function";
+      PeekRange = getFunctionPeekRange(D2);
+    } else if (TagDecl *D2 = dyn_cast<TagDecl>(D)) {
+      Kind = D2->isThisDeclarationADefinition() ? "def" : "decl";
+      PrettyKind = "type";
+
+      if (D2->isThisDeclarationADefinition() && D2->getDefinition() == D2) {
+        PeekRange = getTagPeekRange(D2);
+      } else {
+        PeekRange = SourceRange();
+      }
+    } else if (isa<TypedefNameDecl>(D)) {
+      Kind = "def";
+      PrettyKind = "type";
+      PeekRange = SourceRange(Loc, Loc);
+    } else if (VarDecl *D2 = dyn_cast<VarDecl>(D)) {
+      if (D2->isLocalVarDeclOrParm()) {
+        Flags = NoCrossref;
+      }
+
+      Kind = D2->isThisDeclarationADefinition() == VarDecl::DeclarationOnly
+                 ? "decl"
+                 : "def";
+      PrettyKind = "variable";
+    } else if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D)) {
+      Kind = "def";
+      PrettyKind = "namespace";
+      PeekRange = SourceRange(Loc, Loc);
+    } else if (isa<FieldDecl>(D)) {
+      Kind = "def";
+      PrettyKind = "field";
+    } else if (isa<EnumConstantDecl>(D)) {
+      Kind = "def";
+      PrettyKind = "enum constant";
+    } else {
+      return true;
+    }
+
+    SourceRange CommentRange = getCommentRange(D);
+    PeekRange = combineRanges(PeekRange, CommentRange);
+    PeekRange = validateRange(Loc, PeekRange);
+
+    std::vector<std::string> Symbols = {getMangledName(CurMangleContext, D)};
+    if (CXXMethodDecl::classof(D)) {
+      Symbols.clear();
+      findOverriddenMethods(dyn_cast<CXXMethodDecl>(D), Symbols);
+    }
+
+    // For destructors, loc points to the ~ character. We want to skip to the
+    // class name.
+    if (isa<CXXDestructorDecl>(D)) {
+      const char *P = SM.getCharacterData(Loc);
+      assert(*p == '~');
+      P++;
+
+      unsigned Skipped = 1;
+      while (*P == ' ' || *P == '\t' || *P == '\r' || *P == '\n') {
+        P++;
+        Skipped++;
+      }
+
+      Loc = Loc.getLocWithOffset(Skipped);
+
+      PrettyKind = "destructor";
+    }
+
+    visitIdentifier(Kind, PrettyKind, getQualifiedName(D), Loc, Symbols,
+                    getContext(D), Flags, PeekRange);
+
+    return true;
+  }
+
+  bool VisitCxxConstructExpr(CXXConstructExpr *E) {
+    SourceLocation Loc = E->getLocStart();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    FunctionDecl *Ctor = E->getConstructor();
+    if (Ctor->isTemplateInstantiation()) {
+      Ctor = Ctor->getTemplateInstantiationPattern();
+    }
+    std::string Mangled = getMangledName(CurMangleContext, Ctor);
+
+    // FIXME: Need to do something different for list initialization.
+
+    visitIdentifier("use", "constructor", getQualifiedName(Ctor), Loc, Mangled,
+                    getContext(Loc));
+
+    return true;
+  }
+
+  bool VisitCallExpr(CallExpr *E) {
+    Decl *Callee = E->getCalleeDecl();
+    if (!Callee || !FunctionDecl::classof(Callee)) {
+      return true;
+    }
+
+    const NamedDecl *NamedCallee = dyn_cast<NamedDecl>(Callee);
+
+    SourceLocation Loc;
+
+    const FunctionDecl *F = dyn_cast<FunctionDecl>(NamedCallee);
+    if (F->isTemplateInstantiation()) {
+      NamedCallee = F->getTemplateInstantiationPattern();
+    }
+
+    std::string Mangled = getMangledName(CurMangleContext, NamedCallee);
+
+    Expr *CalleeExpr = E->getCallee()->IgnoreParenImpCasts();
+
+    if (CXXOperatorCallExpr::classof(E)) {
+      // Just take the first token.
+      CXXOperatorCallExpr *Op = dyn_cast<CXXOperatorCallExpr>(E);
+      Loc = Op->getOperatorLoc();
+    } else if (MemberExpr::classof(CalleeExpr)) {
+      MemberExpr *Member = dyn_cast<MemberExpr>(CalleeExpr);
+      Loc = Member->getMemberLoc();
+    } else if (DeclRefExpr::classof(CalleeExpr)) {
+      // We handle this in VisitDeclRefExpr.
+      return true;
+    } else {
+      return true;
+    }
+
+    normalizeLocation(&Loc);
+
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    visitIdentifier("use", "function", getQualifiedName(NamedCallee), Loc, Mangled,
+                    getContext(Loc));
+
+    return true;
+  }
+
+  bool VisitTagTypeLoc(TagTypeLoc L) {
+    SourceLocation Loc = L.getBeginLoc();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    TagDecl *Decl = L.getDecl();
+    std::string Mangled = getMangledName(CurMangleContext, Decl);
+    visitIdentifier("use", "type", getQualifiedName(Decl), Loc, Mangled,
+                    getContext(Loc));
+    return true;
+  }
+
+  bool VisitTypedefTypeLoc(TypedefTypeLoc L) {
+    SourceLocation Loc = L.getBeginLoc();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    NamedDecl *Decl = L.getTypedefNameDecl();
+    std::string Mangled = getMangledName(CurMangleContext, Decl);
+    visitIdentifier("use", "type", getQualifiedName(Decl), Loc, Mangled,
+                    getContext(Loc));
+    return true;
+  }
+
+  bool VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc L) {
+    SourceLocation Loc = L.getBeginLoc();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    NamedDecl *Decl = L.getDecl();
+    std::string Mangled = getMangledName(CurMangleContext, Decl);
+    visitIdentifier("use", "type", getQualifiedName(Decl), Loc, Mangled,
+                    getContext(Loc));
+    return true;
+  }
+
+  bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc L) {
+    SourceLocation Loc = L.getBeginLoc();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    TemplateDecl *Td = L.getTypePtr()->getTemplateName().getAsTemplateDecl();
+    if (ClassTemplateDecl *D = dyn_cast<ClassTemplateDecl>(Td)) {
+      NamedDecl *Decl = D->getTemplatedDecl();
+      std::string Mangled = getMangledName(CurMangleContext, Decl);
+      visitIdentifier("use", "type", getQualifiedName(Decl), Loc, Mangled,
+                      getContext(Loc));
+    } else if (TypeAliasTemplateDecl *D = dyn_cast<TypeAliasTemplateDecl>(Td)) {
+      NamedDecl *Decl = D->getTemplatedDecl();
+      std::string Mangled = getMangledName(CurMangleContext, Decl);
+      visitIdentifier("use", "type", getQualifiedName(Decl), Loc, Mangled,
+                      getContext(Loc));
+    }
+
+    return true;
+  }
+
+  bool VisitDeclRefExpr(DeclRefExpr *E) {
+    SourceLocation Loc = E->getExprLoc();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    if (E->hasQualifier()) {
+      Loc = E->getNameInfo().getLoc();
+      normalizeLocation(&Loc);
+    }
+
+    NamedDecl *Decl = E->getDecl();
+    if (const VarDecl *D2 = dyn_cast<VarDecl>(Decl)) {
+      int Flags = 0;
+      if (D2->isLocalVarDeclOrParm()) {
+        Flags = NoCrossref;
+      }
+      std::string Mangled = getMangledName(CurMangleContext, Decl);
+      visitIdentifier("use", "variable", getQualifiedName(Decl), Loc, Mangled,
+                      getContext(Loc), Flags);
+    } else if (isa<FunctionDecl>(Decl)) {
+      const FunctionDecl *F = dyn_cast<FunctionDecl>(Decl);
+      if (F->isTemplateInstantiation()) {
+        Decl = F->getTemplateInstantiationPattern();
+      }
+
+      std::string Mangled = getMangledName(CurMangleContext, Decl);
+      visitIdentifier("use", "function", getQualifiedName(Decl), Loc, Mangled,
+                      getContext(Loc));
+    } else if (isa<EnumConstantDecl>(Decl)) {
+      std::string Mangled = getMangledName(CurMangleContext, Decl);
+      visitIdentifier("use", "enum", getQualifiedName(Decl), Loc, Mangled,
+                      getContext(Loc));
+    }
+
+    return true;
+  }
+
+  bool VisitCxxConstructorDecl(CXXConstructorDecl *D) {
+    if (!isInterestingLocation(D->getLocation())) {
+      return true;
+    }
+
+    for (CXXConstructorDecl::init_const_iterator It = D->init_begin();
+         It != D->init_end(); ++It) {
+      const CXXCtorInitializer *Ci = *It;
+      if (!Ci->getMember() || !Ci->isWritten()) {
+        continue;
+      }
+
+      SourceLocation Loc = Ci->getMemberLocation();
+      normalizeLocation(&Loc);
+      if (!isInterestingLocation(Loc)) {
+        continue;
+      }
+
+      FieldDecl *Member = Ci->getMember();
+      std::string Mangled = getMangledName(CurMangleContext, Member);
+      visitIdentifier("use", "field", getQualifiedName(Member), Loc, Mangled,
+                      getContext(D));
+    }
+
+    return true;
+  }
+
+  bool VisitMemberExpr(MemberExpr *E) {
+    SourceLocation Loc = E->getExprLoc();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    ValueDecl *Decl = E->getMemberDecl();
+    if (FieldDecl *Field = dyn_cast<FieldDecl>(Decl)) {
+      std::string Mangled = getMangledName(CurMangleContext, Field);
+      visitIdentifier("use", "field", getQualifiedName(Field), Loc, Mangled,
+                      getContext(Loc));
+    }
+    return true;
+  }
+
+  bool VisitCxxDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {
+    SourceLocation Loc = E->getMemberLoc();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return true;
+    }
+
+    if (TemplateStack) {
+      TemplateStack->visitDependent(Loc);
+    }
+    return true;
+  }
+
+  void macroDefined(const Token &Tok, const MacroDirective *Macro) {
+    if (Macro->getMacroInfo()->isBuiltinMacro()) {
+      return;
+    }
+    SourceLocation Loc = Tok.getLocation();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return;
+    }
+
+    IdentifierInfo *Ident = Tok.getIdentifierInfo();
+    if (Ident) {
+      std::string Mangled =
+          std::string("M_") + mangleLocation(Loc, Ident->getName());
+      visitIdentifier("def", "macro", Ident->getName(), Loc, Mangled);
+    }
+  }
+
+  void macroUsed(const Token &Tok, const MacroInfo *Macro) {
+    if (!Macro) {
+      return;
+    }
+    if (Macro->isBuiltinMacro()) {
+      return;
+    }
+    SourceLocation Loc = Tok.getLocation();
+    normalizeLocation(&Loc);
+    if (!isInterestingLocation(Loc)) {
+      return;
+    }
+
+    IdentifierInfo *Ident = Tok.getIdentifierInfo();
+    if (Ident) {
+      std::string Mangled =
+          std::string("M_") +
+          mangleLocation(Macro->getDefinitionLoc(), Ident->getName());
+      visitIdentifier("use", "macro", Ident->getName(), Loc, Mangled);
+    }
+  }
+};
+
+void PreprocessorHook::MacroDefined(const Token &Tok,
+                                    const MacroDirective *Md) {
+  Indexer->macroDefined(Tok, Md);
+}
+
+void PreprocessorHook::MacroExpands(const Token &Tok, const MacroDefinition &Md,
+                                    SourceRange Range, const MacroArgs *Ma) {
+  Indexer->macroUsed(Tok, Md.getMacroInfo());
+}
+
+#if CLANG_VERSION_MAJOR >= 5
+void PreprocessorHook::MacroUndefined(const Token &tok,
+                                      const MacroDefinition &md,
+                                      const MacroDirective *Undef)
+#else
+void PreprocessorHook::MacroUndefined(const Token &Tok,
+                                      const MacroDefinition &Md)
+#endif
+{
+  Indexer->macroUsed(Tok, Md.getMacroInfo());
+}
+
+void PreprocessorHook::Defined(const Token &Tok, const MacroDefinition &Md,
+                               SourceRange Range) {
+  Indexer->macroUsed(Tok, Md.getMacroInfo());
+}
+
+void PreprocessorHook::Ifdef(SourceLocation Loc, const Token &Tok,
+                             const MacroDefinition &Md) {
+  Indexer->macroUsed(Tok, Md.getMacroInfo());
+}
+
+void PreprocessorHook::Ifndef(SourceLocation Loc, const Token &Tok,
+                              const MacroDefinition &Md) {
+  Indexer->macroUsed(Tok, Md.getMacroInfo());
+}
+
+class IndexAction : public PluginASTAction {
+protected:
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 llvm::StringRef F) {
+    return llvm::make_unique<IndexConsumer>(CI);
+  }
+
+  bool ParseArgs(const CompilerInstance &CI,
+                 const std::vector<std::string> &Args) {
+    if (Args.size() != 3) {
+      DiagnosticsEngine &D = CI.getDiagnostics();
+      unsigned DiagID = D.getCustomDiagID(
+          DiagnosticsEngine::Error,
+          "Need arguments for the source, output, and object directories");
+      D.Report(DiagID);
+      return false;
+    }
+
+    // Load our directories
+    Srcdir = getAbsolutePath(Args[0]);
+    if (Srcdir.empty()) {
+      DiagnosticsEngine &D = CI.getDiagnostics();
+      unsigned DiagID = D.getCustomDiagID(
+          DiagnosticsEngine::Error, "Source directory '%0' does not exist");
+      D.Report(DiagID) << Args[0];
+      return false;
+    }
+
+    ensurePath(Args[1] + "/");
+    Outdir = getAbsolutePath(Args[1]);
+    Outdir += "/";
+
+    Objdir = getAbsolutePath(Args[2]);
+    if (Objdir.empty()) {
+      DiagnosticsEngine &D = CI.getDiagnostics();
+      unsigned DiagID = D.getCustomDiagID(DiagnosticsEngine::Error,
+                                          "Objdir '%0' does not exist");
+      D.Report(DiagID) << Args[2];
+      return false;
+    }
+    Objdir += "/";
+
+    printf("MOZSEARCH: %s %s %s\n", Srcdir.c_str(), Outdir.c_str(),
+           Objdir.c_str());
+
+    return true;
+  }
+
+  void printHelp(llvm::raw_ostream &Ros) {
+    Ros << "Help for mozsearch plugin goes here\n";
+  }
+};
+
+static FrontendPluginRegistry::Add<IndexAction>
+    Y("mozsearch-index", "create the mozsearch index database");
new file mode 100644
--- /dev/null
+++ b/build/clang-plugin/mozsearch-plugin/README
@@ -0,0 +1,12 @@
+This clang plugin code generates a JSON file for each compiler input
+file. The JSON file contains information about the C++ symbols that
+are referenced by the input file. The data is eventually consumed by
+Searchfox. See https://github.com/bill-mccloskey/mozsearch for more
+information.
+
+This plugin is enabled with the --enable-clang-plugin and
+--enable-mozsearch-plugin mozconfig options. The output of the plugin
+is stored in $OBJDIR/mozsearch_index.
+
+This code is not a checker, unlike other parts of the Mozilla clang
+plugin. It cannot be used with clang-tidy.
new file mode 100644
--- /dev/null
+++ b/build/clang-plugin/mozsearch-plugin/StringOperations.cpp
@@ -0,0 +1,42 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "StringOperations.h"
+
+static unsigned long djbHash(const char *Str) {
+  unsigned long Hash = 5381;
+
+  for (const char *P = Str; *P; P++) {
+    // Hash * 33 + c
+    Hash = ((Hash << 5) + Hash) + *P;
+  }
+
+  return Hash;
+}
+
+// This doesn't actually return a hex string of |hash|, but it
+// does... something. It doesn't really matter what.
+static void hashToString(unsigned long Hash, char *Buffer) {
+  const char Table[] = {"0123456789abcdef"};
+  char *P = Buffer;
+  while (Hash) {
+    *P = Table[Hash & 0xf];
+    Hash >>= 4;
+    P++;
+  }
+
+  *P = 0;
+}
+
+std::string hash(const std::string &Str) {
+  static char HashStr[41];
+  unsigned long H = djbHash(Str.c_str());
+  hashToString(H, HashStr);
+  return std::string(HashStr);
+}
+
+std::string toString(int N) {
+  return stringFormat("%d", N);
+}
new file mode 100644
--- /dev/null
+++ b/build/clang-plugin/mozsearch-plugin/StringOperations.h
@@ -0,0 +1,25 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef StringOperations_h
+#define StringOperations_h
+
+#include <memory>
+#include <string>
+#include <string.h>
+
+std::string hash(const std::string &Str);
+
+template <typename... Args>
+inline std::string stringFormat(const std::string &Format, Args... ArgList) {
+  size_t Len = snprintf(nullptr, 0, Format.c_str(), ArgList...);
+  std::unique_ptr<char[]> Buf(new char[Len + 1]);
+  snprintf(Buf.get(), Len + 1, Format.c_str(), ArgList...);
+  return std::string(Buf.get(), Buf.get() + Len);
+}
+
+std::string toString(int N);
+
+#endif
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@@ -1509,16 +1509,22 @@ set_config('LINKER_LDFLAGS', select_link
 
 
 js_option('--enable-clang-plugin', env='ENABLE_CLANG_PLUGIN',
           help="Enable building with the mozilla clang plugin")
 
 add_old_configure_assignment('ENABLE_CLANG_PLUGIN',
                              depends_if('--enable-clang-plugin')(lambda _: True))
 
+js_option('--enable-mozsearch-plugin', env='ENABLE_MOZSEARCH_PLUGIN',
+          help="Enable building with the mozsearch indexer plugin")
+
+add_old_configure_assignment('ENABLE_MOZSEARCH_PLUGIN',
+                             depends_if('--enable-mozsearch-plugin')(lambda _: True))
+
 # Code Coverage
 # ==============================================================
 
 js_option('--enable-coverage', env='MOZ_CODE_COVERAGE',
           help='Enable code coverage')
 
 
 @depends('--enable-coverage')
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -81,16 +81,19 @@ SYMBOL_FULL_ARCHIVE_BASENAME = $(PKG_BAS
 SYMBOL_ARCHIVE_BASENAME = $(PKG_BASENAME).crashreporter-symbols
 
 # Generated file package naming
 GENERATED_SOURCE_FILE_PACKAGE = $(PKG_BASENAME).generated-files.tar.gz
 
 # Code coverage package naming
 CODE_COVERAGE_ARCHIVE_BASENAME = $(PKG_BASENAME).code-coverage-gcno
 
+# Mozsearch package naming
+MOZSEARCH_ARCHIVE_BASENAME = $(PKG_BASENAME).mozsearch-index
+
 # Mozharness naming
 MOZHARNESS_PACKAGE = mozharness.zip
 
 # Test package naming
 TEST_PACKAGE = $(PKG_BASENAME).common.tests.zip
 CPP_TEST_PACKAGE = $(PKG_BASENAME).cppunittest.tests.zip
 XPC_TEST_PACKAGE = $(PKG_BASENAME).xpcshell.tests.zip
 MOCHITEST_PACKAGE = $(PKG_BASENAME).mochitest.tests.zip
--- a/toolkit/mozapps/installer/packager.mk
+++ b/toolkit/mozapps/installer/packager.mk
@@ -72,16 +72,22 @@ ifdef MOZ_CODE_COVERAGE
 	$(topsrcdir)/mach build-backend -b ChromeMap
 	@echo 'Packaging code coverage data...'
 	# Package code coverage gcno tree
 	@echo 'Packaging code coverage data...'
 	$(RM) $(CODE_COVERAGE_ARCHIVE_BASENAME).zip
 	$(PYTHON) -mmozbuild.codecoverage.packager \
 		--output-file='$(DIST)/$(PKG_PATH)$(CODE_COVERAGE_ARCHIVE_BASENAME).zip'
 endif
+ifdef ENABLE_MOZSEARCH_PLUGIN
+	@echo 'Generating mozsearch tarball...'
+	$(RM) $(MOZSEARCH_ARCHIVE_BASENAME).zip
+	cd $(topobjdir)/mozsearch_index && \
+          zip -r5D '$(ABS_DIST)/$(PKG_PATH)$(MOZSEARCH_ARCHIVE_BASENAME).zip' .
+endif
 ifeq (Darwin, $(OS_ARCH))
 ifdef MOZ_ASAN
 	@echo "Rewriting ASan runtime dylib paths for all binaries in $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH) ..."
 	$(PYTHON) $(MOZILLA_DIR)/build/unix/rewrite_asan_dylib.py $(DIST)/$(MOZ_PKG_DIR)$(_BINPATH)
 endif # MOZ_ASAN
 endif # Darwin
 ifdef MOZ_STYLO
 ifndef MOZ_ARTIFACT_BUILDS
--- a/toolkit/mozapps/installer/upload-files.mk
+++ b/toolkit/mozapps/installer/upload-files.mk
@@ -424,16 +424,20 @@ ifdef MOZ_CODE_COVERAGE
     $(call QUOTED_WILDCARD,$(topobjdir)/chrome-map.json) \
     $(NULL)
 endif
 
 ifdef MOZ_STYLO
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(STYLO_BINDINGS_PACKAGE))
 endif
 
+ifdef ENABLE_MOZSEARCH_PLUGIN
+  UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(PKG_PATH)$(MOZSEARCH_ARCHIVE_BASENAME).zip)
+endif
+
 SIGN_CHECKSUM_CMD=
 ifdef MOZ_SIGN_CMD
   # If we're signing with gpg, we'll have a bunch of extra detached signatures to
   # upload. We also want to sign our checksums file
   SIGN_CHECKSUM_CMD=$(MOZ_SIGN_CMD) -f gpg $(CHECKSUM_FILE)
 
   CHECKSUM_FILES += $(CHECKSUM_FILE).asc
   UPLOAD_FILES += $(call QUOTED_WILDCARD,$(DIST)/$(COMPLETE_MAR).asc)