Imported Upstream version 6.10.0.49

Former-commit-id: 1d6753294b2993e1fbf92de9366bb9544db4189b

external/llvm-project/clang-tools-extra/clangd/index/FileIndex.cpp

@@ -0,0 +1,84 @@
+//===--- FileIndex.cpp - Indexes for files. ------------------------ C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "FileIndex.h"
+#include "SymbolCollector.h"
+#include "clang/Index/IndexingAction.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+/// Retrieves namespace and class level symbols in \p Decls.
+std::unique_ptr<SymbolSlab> indexAST(ASTContext &Ctx,
+                                     llvm::ArrayRef<const Decl *> Decls) {
+  auto Collector = std::make_shared<SymbolCollector>();
+  index::IndexingOptions IndexOpts;
+  IndexOpts.SystemSymbolFilter =
+      index::IndexingOptions::SystemSymbolFilterKind::All;
+  IndexOpts.IndexFunctionLocals = false;
+
+  index::indexTopLevelDecls(Ctx, Decls, Collector, IndexOpts);
+  auto Symbols = llvm::make_unique<SymbolSlab>();
+  *Symbols = Collector->takeSymbols();
+  return Symbols;
+}
+
+} // namespace
+
+void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Slab) {
+  std::lock_guard<std::mutex> Lock(Mutex);
+  if (!Slab)
+    FileToSlabs.erase(Path);
+  else
+    FileToSlabs[Path] = std::move(Slab);
+}
+
+std::shared_ptr<std::vector<const Symbol *>> FileSymbols::allSymbols() {
+  // The snapshot manages life time of symbol slabs and provides pointers of all
+  // symbols in all slabs.
+  struct Snapshot {
+    std::vector<const Symbol *> Pointers;
+    std::vector<std::shared_ptr<SymbolSlab>> KeepAlive;
+  };
+  auto Snap = std::make_shared<Snapshot>();
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    for (const auto &FileAndSlab : FileToSlabs) {
+      Snap->KeepAlive.push_back(FileAndSlab.second);
+      for (const auto &Iter : *FileAndSlab.second)
+        Snap->Pointers.push_back(&Iter);
+    }
+  }
+  auto *Pointers = &Snap->Pointers;
+  // Use aliasing constructor to keep the snapshot alive along with the
+  // pointers.
+  return {std::move(Snap), Pointers};
+}
+
+void FileIndex::update(const Context &Ctx, PathRef Path, ParsedAST *AST) {
+  if (!AST) {
+    FSymbols.update(Path, nullptr);
+  } else {
+    auto Slab = indexAST(AST->getASTContext(), AST->getTopLevelDecls());
+    FSymbols.update(Path, std::move(Slab));
+  }
+  auto Symbols = FSymbols.allSymbols();
+  Index.build(std::move(Symbols));
+}
+
+bool FileIndex::fuzzyFind(
+    const Context &Ctx, const FuzzyFindRequest &Req,
+    llvm::function_ref<void(const Symbol &)> Callback) const {
+  return Index.fuzzyFind(Ctx, Req, Callback);
+}
+
+} // namespace clangd
+} // namespace clang

external/llvm-project/clang-tools-extra/clangd/index/FileIndex.h

@@ -0,0 +1,75 @@
+//===--- FileIndex.h - Index for files. ---------------------------- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// FileIndex implements SymbolIndex for symbols from a set of files. Symbols are
+// maintained at source-file granuality (e.g. with ASTs), and files can be
+// updated dynamically.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H
+
+#include "../ClangdUnit.h"
+#include "../Context.h"
+#include "Index.h"
+#include "MemIndex.h"
+
+namespace clang {
+namespace clangd {
+
+/// \brief A container of Symbols from several source files. It can be updated
+/// at source-file granularity, replacing all symbols from one file with a new
+/// set.
+///
+/// This implements a snapshot semantics for symbols in a file. Each update to a
+/// file will create a new snapshot for all symbols in the file. Snapshots are
+/// managed with shared pointers that are shared between this class and the
+/// users. For each file, this class only stores a pointer pointing to the
+/// newest snapshot, and an outdated snapshot is deleted by the last owner of
+/// the snapshot, either this class or the symbol index.
+///
+/// The snapshot semantics keeps critical sections minimal since we only need
+/// locking when we swap or obtain refereces to snapshots.
+class FileSymbols {
+public:
+  /// \brief Updates all symbols in a file. If \p Slab is nullptr, symbols for
+  /// \p Path will be removed.
+  void update(PathRef Path, std::unique_ptr<SymbolSlab> Slab);
+
+  // The shared_ptr keeps the symbols alive
+  std::shared_ptr<std::vector<const Symbol *>> allSymbols();
+
+private:
+  mutable std::mutex Mutex;
+
+  /// \brief Stores the latest snapshots for all active files.
+  llvm::StringMap<std::shared_ptr<SymbolSlab>> FileToSlabs;
+};
+
+/// \brief This manages symbls from files and an in-memory index on all symbols.
+class FileIndex : public SymbolIndex {
+public:
+  /// \brief Update symbols in \p Path with symbols in \p AST. If \p AST is
+  /// nullptr, this removes all symbols in the file
+  void update(const Context &Ctx, PathRef Path, ParsedAST *AST);
+
+  bool
+  fuzzyFind(const Context &Ctx, const FuzzyFindRequest &Req,
+            llvm::function_ref<void(const Symbol &)> Callback) const override;
+
+private:
+  FileSymbols FSymbols;
+  MemIndex Index;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H

external/llvm-project/clang-tools-extra/clangd/index/Index.cpp

@@ -0,0 +1,85 @@
+//===--- Index.cpp -----------------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Index.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/SHA1.h"
+
+namespace clang {
+namespace clangd {
+using namespace llvm;
+
+SymbolID::SymbolID(StringRef USR)
+    : HashValue(SHA1::hash(arrayRefFromStringRef(USR))) {}
+
+raw_ostream &operator<<(raw_ostream &OS, const SymbolID &ID) {
+  OS << toHex(toStringRef(ID.HashValue));
+  return OS;
+}
+
+void operator>>(StringRef Str, SymbolID &ID) {
+  std::string HexString = fromHex(Str);
+  assert(HexString.size() == ID.HashValue.size());
+  std::copy(HexString.begin(), HexString.end(), ID.HashValue.begin());
+}
+
+SymbolSlab::const_iterator SymbolSlab::find(const SymbolID &ID) const {
+  auto It = std::lower_bound(Symbols.begin(), Symbols.end(), ID,
+                             [](const Symbol &S, const SymbolID &I) {
+                               return S.ID < I;
+                             });
+  if (It != Symbols.end() && It->ID == ID)
+    return It;
+  return Symbols.end();
+}
+
+// Copy the underlying data of the symbol into the owned arena.
+static void own(Symbol &S, DenseSet<StringRef> &Strings,
+                BumpPtrAllocator &Arena) {
+  // Intern replaces V with a reference to the same string owned by the arena.
+  auto Intern = [&](StringRef &V) {
+    auto R = Strings.insert(V);
+    if (R.second) { // New entry added to the table, copy the string.
+      *R.first = V.copy(Arena);
+    }
+    V = *R.first;
+  };
+
+  // We need to copy every StringRef field onto the arena.
+  Intern(S.Name);
+  Intern(S.Scope);
+  Intern(S.CanonicalDeclaration.FilePath);
+}
+
+void SymbolSlab::Builder::insert(const Symbol &S) {
+  auto R = SymbolIndex.try_emplace(S.ID, Symbols.size());
+  if (R.second) {
+    Symbols.push_back(S);
+    own(Symbols.back(), Strings, Arena);
+  } else {
+    auto &Copy = Symbols[R.first->second] = S;
+    own(Copy, Strings, Arena);
+  }
+}
+
+SymbolSlab SymbolSlab::Builder::build() && {
+  Symbols = {Symbols.begin(), Symbols.end()}; // Force shrink-to-fit.
+  // Sort symbols so the slab can binary search over them.
+  std::sort(Symbols.begin(), Symbols.end(),
+            [](const Symbol &L, const Symbol &R) { return L.ID < R.ID; });
+  // We may have unused strings from overwritten symbols. Build a new arena.
+  BumpPtrAllocator NewArena;
+  DenseSet<StringRef> Strings;
+  for (auto &S : Symbols)
+    own(S, Strings, NewArena);
+  return SymbolSlab(std::move(NewArena), std::move(Symbols));
+}
+
+} // namespace clangd
+} // namespace clang

external/llvm-project/clang-tools-extra/clangd/index/Index.h

@@ -0,0 +1,226 @@
+//===--- Symbol.h -----------------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H
+
+#include "../Context.h"
+#include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/StringExtras.h"
+#include <array>
+#include <string>
+
+namespace clang {
+namespace clangd {
+
+struct SymbolLocation {
+  // The absolute path of the source file where a symbol occurs.
+  llvm::StringRef FilePath;
+  // The 0-based offset to the first character of the symbol from the beginning
+  // of the source file.
+  unsigned StartOffset;
+  // The 0-based offset to the last character of the symbol from the beginning
+  // of the source file.
+  unsigned EndOffset;
+};
+
+// The class identifies a particular C++ symbol (class, function, method, etc).
+//
+// As USRs (Unified Symbol Resolution) could be large, especially for functions
+// with long type arguments, SymbolID is using 160-bits SHA1(USR) values to
+// guarantee the uniqueness of symbols while using a relatively small amount of
+// memory (vs storing USRs directly).
+//
+// SymbolID can be used as key in the symbol indexes to lookup the symbol.
+class SymbolID {
+public:
+  SymbolID() = default;
+  SymbolID(llvm::StringRef USR);
+
+  bool operator==(const SymbolID &Sym) const {
+    return HashValue == Sym.HashValue;
+  }
+  bool operator<(const SymbolID &Sym) const {
+    return HashValue < Sym.HashValue;
+  }
+
+private:
+  friend llvm::hash_code hash_value(const SymbolID &ID) {
+    // We already have a good hash, just return the first bytes.
+    static_assert(sizeof(size_t) <= 20, "size_t longer than SHA1!");
+    return *reinterpret_cast<const size_t *>(ID.HashValue.data());
+  }
+  friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+                                       const SymbolID &ID);
+  friend void operator>>(llvm::StringRef Str, SymbolID &ID);
+
+  std::array<uint8_t, 20> HashValue;
+};
+
+// Write SymbolID into the given stream. SymbolID is encoded as a 40-bytes
+// hex string.
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const SymbolID &ID);
+
+// Construct SymbolID from a hex string.
+// The HexStr is required to be a 40-bytes hex string, which is encoded from the
+// "<<" operator.
+void operator>>(llvm::StringRef HexStr, SymbolID &ID);
+
+} // namespace clangd
+} // namespace clang
+namespace llvm {
+// Support SymbolIDs as DenseMap keys.
+template <> struct DenseMapInfo<clang::clangd::SymbolID> {
+  static inline clang::clangd::SymbolID getEmptyKey() {
+    static clang::clangd::SymbolID EmptyKey("EMPTYKEY");
+    return EmptyKey;
+  }
+  static inline clang::clangd::SymbolID getTombstoneKey() {
+    static clang::clangd::SymbolID TombstoneKey("TOMBSTONEKEY");
+    return TombstoneKey;
+  }
+  static unsigned getHashValue(const clang::clangd::SymbolID &Sym) {
+    return hash_value(Sym);
+  }
+  static bool isEqual(const clang::clangd::SymbolID &LHS,
+                      const clang::clangd::SymbolID &RHS) {
+    return LHS == RHS;
+  }
+};
+} // namespace llvm
+namespace clang {
+namespace clangd {
+
+// The class presents a C++ symbol, e.g. class, function.
+//
+// WARNING: Symbols do not own much of their underlying data - typically strings
+// are owned by a SymbolSlab. They should be treated as non-owning references.
+// Copies are shallow.
+// When adding new unowned data fields to Symbol, remember to update
+// SymbolSlab::Builder in Index.cpp to copy them to the slab's storage.
+struct Symbol {
+  // The ID of the symbol.
+  SymbolID ID;
+  // The symbol information, like symbol kind.
+  index::SymbolInfo SymInfo;
+  // The unqualified name of the symbol, e.g. "bar" (for "n1::n2::bar").
+  llvm::StringRef Name;
+  // The scope (e.g. namespace) of the symbol, e.g. "n1::n2" (for
+  // "n1::n2::bar").
+  llvm::StringRef Scope;
+  // The location of the canonical declaration of the symbol.
+  //
+  // A C++ symbol could have multiple declarations and one definition (e.g.
+  // a function is declared in ".h" file, and is defined in ".cc" file).
+  //   * For classes, the canonical declaration is usually definition.
+  //   * For non-inline functions, the canonical declaration is a declaration
+  //     (not a definition), which is usually declared in ".h" file.
+  SymbolLocation CanonicalDeclaration;
+
+  // FIXME: add definition location of the symbol.
+  // FIXME: add all occurrences support.
+  // FIXME: add extra fields for index scoring signals.
+  // FIXME: add code completion information.
+};
+
+// An immutable symbol container that stores a set of symbols.
+// The container will maintain the lifetime of the symbols.
+class SymbolSlab {
+public:
+  using const_iterator = std::vector<Symbol>::const_iterator;
+
+  SymbolSlab() = default;
+
+  const_iterator begin() const { return Symbols.begin(); }
+  const_iterator end() const { return Symbols.end(); }
+  const_iterator find(const SymbolID &SymID) const;
+
+  size_t size() const { return Symbols.size(); }
+  // Estimates the total memory usage.
+  size_t bytes() const {
+    return sizeof(*this) + Arena.getTotalMemory() +
+           Symbols.capacity() * sizeof(Symbol);
+  }
+
+  // SymbolSlab::Builder is a mutable container that can 'freeze' to SymbolSlab.
+  // The frozen SymbolSlab will use less memory.
+  class Builder {
+   public:
+     // Adds a symbol, overwriting any existing one with the same ID.
+     // This is a deep copy: underlying strings will be owned by the slab.
+     void insert(const Symbol& S);
+
+     // Returns the symbol with an ID, if it exists. Valid until next insert().
+     const Symbol* find(const SymbolID &ID) {
+       auto I = SymbolIndex.find(ID);
+       return I == SymbolIndex.end() ? nullptr : &Symbols[I->second];
+     }
+
+     // Consumes the builder to finalize the slab.
+     SymbolSlab build() &&;
+
+   private:
+     llvm::BumpPtrAllocator Arena;
+     // Intern table for strings. Contents are on the arena.
+     llvm::DenseSet<llvm::StringRef> Strings;
+     std::vector<Symbol> Symbols;
+     // Values are indices into Symbols vector.
+     llvm::DenseMap<SymbolID, size_t> SymbolIndex;
+  };
+
+private:
+  SymbolSlab(llvm::BumpPtrAllocator Arena, std::vector<Symbol> Symbols)
+      : Arena(std::move(Arena)), Symbols(std::move(Symbols)) {}
+
+  llvm::BumpPtrAllocator Arena; // Owns Symbol data that the Symbols do not.
+  std::vector<Symbol> Symbols;  // Sorted by SymbolID to allow lookup.
+};
+
+struct FuzzyFindRequest {
+  /// \brief A query string for the fuzzy find. This is matched against symbols'
+  /// un-qualified identifiers and should not contain qualifiers like "::".
+  std::string Query;
+  /// \brief If this is non-empty, symbols must be in at least one of the scopes
+  /// (e.g. namespaces) excluding nested scopes. For example, if a scope "xyz"
+  /// is provided, the matched symbols must be defined in scope "xyz" but not
+  /// "xyz::abc".
+  ///
+  /// A scope must be fully qualified without leading or trailing "::" e.g.
+  /// "n1::n2". "" is interpreted as the global namespace, and "::" is invalid.
+  std::vector<std::string> Scopes;
+  /// \brief The maxinum number of candidates to return.
+  size_t MaxCandidateCount = UINT_MAX;
+};
+
+/// \brief Interface for symbol indexes that can be used for searching or
+/// matching symbols among a set of symbols based on names or unique IDs.
+class SymbolIndex {
+public:
+  virtual ~SymbolIndex() = default;
+
+  /// \brief Matches symbols in the index fuzzily and applies \p Callback on
+  /// each matched symbol before returning.
+  ///
+  /// Returns true if the result list is complete, false if it was truncated due
+  /// to MaxCandidateCount
+  virtual bool
+  fuzzyFind(const Context &Ctx, const FuzzyFindRequest &Req,
+            llvm::function_ref<void(const Symbol &)> Callback) const = 0;
+
+  // FIXME: add interfaces for more index use cases:
+  //  - Symbol getSymbolInfo(SymbolID);
+  //  - getAllOccurrences(SymbolID);
+};
+
+} // namespace clangd
+} // namespace clang
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_INDEX_H

external/llvm-project/clang-tools-extra/clangd/index/MemIndex.cpp

@@ -0,0 +1,57 @@
+//===--- MemIndex.cpp - Dynamic in-memory symbol index. ----------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===-------------------------------------------------------------------===//
+
+#include "MemIndex.h"
+#include "../Logger.h"
+
+namespace clang {
+namespace clangd {
+
+void MemIndex::build(std::shared_ptr<std::vector<const Symbol *>> Syms) {
+  llvm::DenseMap<SymbolID, const Symbol *> TempIndex;
+  for (const Symbol *Sym : *Syms)
+    TempIndex[Sym->ID] = Sym;
+
+  // Swap out the old symbols and index.
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    Index = std::move(TempIndex);
+    Symbols = std::move(Syms); // Relase old symbols.
+  }
+}
+
+bool MemIndex::fuzzyFind(
+    const Context &Ctx, const FuzzyFindRequest &Req,
+    llvm::function_ref<void(const Symbol &)> Callback) const {
+  assert(!StringRef(Req.Query).contains("::") &&
+         "There must be no :: in query.");
+
+  unsigned Matched = 0;
+  {
+    std::lock_guard<std::mutex> Lock(Mutex);
+    for (const auto Pair : Index) {
+      const Symbol *Sym = Pair.second;
+
+      // Exact match against all possible scopes.
+      if (!Req.Scopes.empty() && !llvm::is_contained(Req.Scopes, Sym->Scope))
+        continue;
+
+      // FIXME(ioeric): use fuzzy matcher.
+      if (StringRef(Sym->Name).find_lower(Req.Query) != StringRef::npos) {
+        if (++Matched > Req.MaxCandidateCount)
+          return false;
+        Callback(*Sym);
+      }
+    }
+  }
+  return true;
+}
+
+} // namespace clangd
+} // namespace clang

external/llvm-project/clang-tools-extra/clangd/index/MemIndex.h

@@ -0,0 +1,42 @@
+//===--- MemIndex.h - Dynamic in-memory symbol index. -------------- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_MEMINDEX_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_MEMINDEX_H
+
+#include "Index.h"
+#include <mutex>
+
+namespace clang {
+namespace clangd {
+
+/// \brief This implements an index for a (relatively small) set of symbols that
+/// can be easily managed in memory.
+class MemIndex : public SymbolIndex {
+public:
+  /// \brief (Re-)Build index for `Symbols`. All symbol pointers must remain
+  /// accessible as long as `Symbols` is kept alive.
+  void build(std::shared_ptr<std::vector<const Symbol *>> Symbols);
+
+  bool
+  fuzzyFind(const Context &Ctx, const FuzzyFindRequest &Req,
+            llvm::function_ref<void(const Symbol &)> Callback) const override;
+
+private:
+  std::shared_ptr<std::vector<const Symbol *>> Symbols;
+  // Index is a set of symbols that are deduplicated by symbol IDs.
+  // FIXME: build smarter index structure.
+  llvm::DenseMap<SymbolID, const Symbol *> Index;
+  mutable std::mutex Mutex;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_MEMINDEX_H

external/llvm-project/clang-tools-extra/clangd/index/SymbolCollector.cpp

@@ -0,0 +1,116 @@
+//===--- SymbolCollector.cpp -------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolCollector.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Index/IndexSymbol.h"
+#include "clang/Index/USRGeneration.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+
+namespace clang {
+namespace clangd {
+
+namespace {
+// Make the Path absolute using the current working directory of the given
+// SourceManager if the Path is not an absolute path.
+//
+// The Path can be a path relative to the build directory, or retrieved from
+// the SourceManager.
+std::string makeAbsolutePath(const SourceManager &SM, StringRef Path) {
+  llvm::SmallString<128> AbsolutePath(Path);
+  if (std::error_code EC =
+          SM.getFileManager().getVirtualFileSystem()->makeAbsolute(
+              AbsolutePath))
+    llvm::errs() << "Warning: could not make absolute file: '" << EC.message()
+                 << '\n';
+  // Handle the symbolic link path case where the current working directory
+  // (getCurrentWorkingDirectory) is a symlink./ We always want to the real
+  // file path (instead of the symlink path) for the  C++ symbols.
+  //
+  // Consider the following example:
+  //
+  //   src dir: /project/src/foo.h
+  //   current working directory (symlink): /tmp/build -> /project/src/
+  //
+  // The file path of Symbol is "/project/src/foo.h" instead of
+  // "/tmp/build/foo.h"
+  const DirectoryEntry *Dir = SM.getFileManager().getDirectory(
+      llvm::sys::path::parent_path(AbsolutePath.str()));
+  if (Dir) {
+    StringRef DirName = SM.getFileManager().getCanonicalName(Dir);
+    SmallString<128> AbsoluteFilename;
+    llvm::sys::path::append(AbsoluteFilename, DirName,
+                            llvm::sys::path::filename(AbsolutePath.str()));
+    return AbsoluteFilename.str();
+  }
+  return AbsolutePath.str();
+}
+
+// Split a qualified symbol name into scope and unqualified name, e.g. given
+// "a::b::c", return {"a::b", "c"}. Scope is empty if it doesn't exist.
+std::pair<llvm::StringRef, llvm::StringRef>
+splitQualifiedName(llvm::StringRef QName) {
+  assert(!QName.startswith("::") && "Qualified names should not start with ::");
+  size_t Pos = QName.rfind("::");
+  if (Pos == llvm::StringRef::npos)
+    return {StringRef(), QName};
+  return {QName.substr(0, Pos), QName.substr(Pos + 2)};
+}
+
+} // namespace
+
+// Always return true to continue indexing.
+bool SymbolCollector::handleDeclOccurence(
+    const Decl *D, index::SymbolRoleSet Roles,
+    ArrayRef<index::SymbolRelation> Relations, FileID FID, unsigned Offset,
+    index::IndexDataConsumer::ASTNodeInfo ASTNode) {
+  // FIXME: collect all symbol references.
+  if (!(Roles & static_cast<unsigned>(index::SymbolRole::Declaration) ||
+        Roles & static_cast<unsigned>(index::SymbolRole::Definition)))
+    return true;
+
+  if (const NamedDecl *ND = llvm::dyn_cast<NamedDecl>(D)) {
+    // FIXME: Should we include the internal linkage symbols?
+    if (!ND->hasExternalFormalLinkage() || ND->isInAnonymousNamespace())
+      return true;
+
+    llvm::SmallString<128> USR;
+    if (index::generateUSRForDecl(ND, USR))
+      return true;
+
+    auto ID = SymbolID(USR);
+    if (Symbols.find(ID) != nullptr)
+      return true;
+
+    auto &SM = ND->getASTContext().getSourceManager();
+    std::string FilePath =
+        makeAbsolutePath(SM, SM.getFilename(D->getLocation()));
+    SymbolLocation Location = {FilePath, SM.getFileOffset(D->getLocStart()),
+                               SM.getFileOffset(D->getLocEnd())};
+    std::string QName = ND->getQualifiedNameAsString();
+    auto ScopeAndName = splitQualifiedName(QName);
+
+    Symbol S;
+    S.ID = std::move(ID);
+    S.Scope = ScopeAndName.first;
+    S.Name = ScopeAndName.second;
+    S.SymInfo = index::getSymbolInfo(D);
+    S.CanonicalDeclaration = Location;
+    Symbols.insert(S);
+  }
+
+  return true;
+}
+
+} // namespace clangd
+} // namespace clang

external/llvm-project/clang-tools-extra/clangd/index/SymbolCollector.h

@@ -0,0 +1,40 @@
+//===--- SymbolCollector.h ---------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Index.h"
+#include "clang/Index/IndexDataConsumer.h"
+#include "clang/Index/IndexSymbol.h"
+
+namespace clang {
+namespace clangd {
+
+// Collect all symbols from an AST.
+//
+// Clients (e.g. clangd) can use SymbolCollector together with
+// index::indexTopLevelDecls to retrieve all symbols when the source file is
+// changed.
+class SymbolCollector : public index::IndexDataConsumer {
+public:
+  SymbolCollector() = default;
+
+  bool
+  handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles,
+                      ArrayRef<index::SymbolRelation> Relations, FileID FID,
+                      unsigned Offset,
+                      index::IndexDataConsumer::ASTNodeInfo ASTNode) override;
+
+  SymbolSlab takeSymbols() { return std::move(Symbols).build(); }
+
+private:
+  // All Symbols collected from the AST.
+  SymbolSlab::Builder Symbols;
+};
+
+} // namespace clangd
+} // namespace clang

external/llvm-project/clang-tools-extra/clangd/index/SymbolYAML.cpp

@@ -0,0 +1,146 @@
+//===--- SymbolYAML.cpp ------------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolYAML.h"
+#include "Index.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(clang::clangd::Symbol)
+
+namespace llvm {
+namespace yaml {
+
+using clang::clangd::Symbol;
+using clang::clangd::SymbolID;
+using clang::clangd::SymbolLocation;
+using clang::index::SymbolInfo;
+using clang::index::SymbolLanguage;
+using clang::index::SymbolKind;
+
+// Helper to (de)serialize the SymbolID. We serialize it as a hex string.
+struct NormalizedSymbolID {
+  NormalizedSymbolID(IO &) {}
+  NormalizedSymbolID(IO &, const SymbolID& ID) {
+    llvm::raw_string_ostream OS(HexString);
+    OS << ID;
+  }
+
+  SymbolID denormalize(IO&) {
+    SymbolID ID;
+    HexString >> ID;
+    return ID;
+  }
+
+  std::string HexString;
+};
+
+template <> struct MappingTraits<SymbolLocation> {
+  static void mapping(IO &IO, SymbolLocation &Value) {
+    IO.mapRequired("StartOffset", Value.StartOffset);
+    IO.mapRequired("EndOffset", Value.EndOffset);
+    IO.mapRequired("FilePath", Value.FilePath);
+  }
+};
+
+template <> struct MappingTraits<SymbolInfo> {
+  static void mapping(IO &io, SymbolInfo &SymInfo) {
+    // FIXME: expose other fields?
+    io.mapRequired("Kind", SymInfo.Kind);
+    io.mapRequired("Lang", SymInfo.Lang);
+  }
+};
+
+template<> struct MappingTraits<Symbol> {
+  static void mapping(IO &IO, Symbol &Sym) {
+    MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(
+        IO, Sym.ID);
+    IO.mapRequired("ID", NSymbolID->HexString);
+    IO.mapRequired("Name", Sym.Name);
+    IO.mapRequired("Scope", Sym.Scope);
+    IO.mapRequired("SymInfo", Sym.SymInfo);
+    IO.mapRequired("CanonicalDeclaration", Sym.CanonicalDeclaration);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolLanguage> {
+  static void enumeration(IO &IO, SymbolLanguage &Value) {
+    IO.enumCase(Value, "C", SymbolLanguage::C);
+    IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
+    IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
+    IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolKind> {
+  static void enumeration(IO &IO, SymbolKind &Value) {
+#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
+
+    DEFINE_ENUM(Unknown);
+    DEFINE_ENUM(Function);
+    DEFINE_ENUM(Module);
+    DEFINE_ENUM(Namespace);
+    DEFINE_ENUM(NamespaceAlias);
+    DEFINE_ENUM(Macro);
+    DEFINE_ENUM(Enum);
+    DEFINE_ENUM(Struct);
+    DEFINE_ENUM(Class);
+    DEFINE_ENUM(Protocol);
+    DEFINE_ENUM(Extension);
+    DEFINE_ENUM(Union);
+    DEFINE_ENUM(TypeAlias);
+    DEFINE_ENUM(Function);
+    DEFINE_ENUM(Variable);
+    DEFINE_ENUM(Field);
+    DEFINE_ENUM(EnumConstant);
+    DEFINE_ENUM(InstanceMethod);
+    DEFINE_ENUM(ClassMethod);
+    DEFINE_ENUM(StaticMethod);
+    DEFINE_ENUM(InstanceProperty);
+    DEFINE_ENUM(ClassProperty);
+    DEFINE_ENUM(StaticProperty);
+    DEFINE_ENUM(Constructor);
+    DEFINE_ENUM(Destructor);
+    DEFINE_ENUM(ConversionFunction);
+    DEFINE_ENUM(Parameter);
+    DEFINE_ENUM(Using);
+
+#undef DEFINE_ENUM
+  }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+namespace clang {
+namespace clangd {
+
+SymbolSlab SymbolFromYAML(llvm::StringRef YAMLContent) {
+  std::vector<Symbol> S;
+  llvm::yaml::Input Yin(YAMLContent);
+  Yin >> S;
+  SymbolSlab::Builder Syms;
+  for (auto& Sym : S)
+    Syms.insert(Sym);
+  return std::move(Syms).build();
+}
+
+std::string SymbolToYAML(const SymbolSlab& Symbols) {
+  std::string Str;
+  llvm::raw_string_ostream OS(Str);
+  llvm::yaml::Output Yout(OS);
+  for (Symbol S : Symbols) // copy: Yout<< requires mutability.
+    Yout<< S;
+  return OS.str();
+}
+
+} // namespace clangd
+} // namespace clang

external/llvm-project/clang-tools-extra/clangd/index/SymbolYAML.h

@@ -0,0 +1,37 @@
+//===--- SymbolYAML.h --------------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// SymbolYAML provides facilities to convert Symbol to YAML, and vice versa.
+// The YAML format of Symbol is designed for simplicity and experiment, but
+// isn't a suitable/efficient store.
+//
+// This is for **experimental** only. Don't use it in the production code.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H
+
+#include "Index.h"
+#include "llvm/Support/Error.h"
+
+namespace clang {
+namespace clangd {
+
+// Read symbols from a YAML-format string.
+SymbolSlab SymbolFromYAML(llvm::StringRef YAMLContent);
+
+// Convert symbols to a YAML-format string.
+// The YAML result is safe to concatenate if you have multiple symbol slabs.
+std::string SymbolToYAML(const SymbolSlab& Symbols);
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H