Imported Upstream version 6.10.0.49

Former-commit-id: 1d6753294b2993e1fbf92de9366bb9544db4189b

external/llvm-project/clang-tools-extra/modularize/CMakeLists.txt

@@ -0,0 +1,26 @@
+set(LLVM_LINK_COMPONENTS
+  Option
+  Support
+  )
+
+add_clang_tool(modularize
+  Modularize.cpp
+  ModuleAssistant.cpp
+  ModularizeUtilities.cpp
+  CoverageChecker.cpp
+  PreprocessorTracker.cpp
+  )
+
+target_link_libraries(modularize
+  PRIVATE
+  clangAST
+  clangBasic
+  clangDriver
+  clangFrontend
+  clangLex
+  clangTooling
+  )
+
+install(TARGETS modularize
+        RUNTIME DESTINATION bin
+        COMPONENT clang-extras)

external/llvm-project/clang-tools-extra/modularize/CoverageChecker.cpp

@@ -0,0 +1,422 @@
+//===--- extra/module-map-checker/CoverageChecker.cpp -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a class that validates a module map by checking that
+// all headers in the corresponding directories are accounted for.
+//
+// This class uses a previously loaded module map object.
+// Starting at the module map file directory, or just the include
+// paths, if specified, it will collect the names of all the files it
+// considers headers (no extension, .h, or .inc--if you need more, modify the
+// ModularizeUtilities::isHeader function).
+//  It then compares the headers against those referenced
+// in the module map, either explicitly named, or implicitly named via an
+// umbrella directory or umbrella file, as parsed by the ModuleMap object.
+// If headers are found which are not referenced or covered by an umbrella
+// directory or file, warning messages will be produced, and the doChecks
+// function will return an error code of 1.  Other errors result in an error
+// code of 2. If no problems are found, an error code of 0 is returned.
+//
+// Note that in the case of umbrella headers, this tool invokes the compiler
+// to preprocess the file, and uses a callback to collect the header files
+// included by the umbrella header or any of its nested includes.  If any
+// front end options are needed for these compiler invocations, these are
+// to be passed in via the CommandLine parameter.
+//
+// Warning message have the form:
+//
+//  warning: module.modulemap does not account for file: Level3A.h
+//
+// Note that for the case of the module map referencing a file that does
+// not exist, the module map parser in Clang will (at the time of this
+// writing) display an error message.
+//
+// Potential problems with this program:
+//
+// 1. Might need a better header matching mechanism, or extensions to the
+//    canonical file format used.
+//
+// 2. It might need to support additional header file extensions.
+//
+// Future directions:
+//
+// 1. Add an option to fix the problems found, writing a new module map.
+//    Include an extra option to add unaccounted-for headers as excluded.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ModularizeUtilities.h"
+#include "clang/AST/ASTConsumer.h"
+#include "CoverageChecker.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Driver/Options.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Option/Option.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace Modularize;
+using namespace clang;
+using namespace clang::driver;
+using namespace clang::driver::options;
+using namespace clang::tooling;
+namespace cl = llvm::cl;
+namespace sys = llvm::sys;
+
+// Preprocessor callbacks.
+// We basically just collect include files.
+class CoverageCheckerCallbacks : public PPCallbacks {
+public:
+  CoverageCheckerCallbacks(CoverageChecker &Checker) : Checker(Checker) {}
+  ~CoverageCheckerCallbacks() override {}
+
+  // Include directive callback.
+  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
+                          StringRef FileName, bool IsAngled,
+                          CharSourceRange FilenameRange, const FileEntry *File,
+                          StringRef SearchPath, StringRef RelativePath,
+                          const Module *Imported) override {
+    Checker.collectUmbrellaHeaderHeader(File->getName());
+  }
+
+private:
+  CoverageChecker &Checker;
+};
+
+// Frontend action stuff:
+
+// Consumer is responsible for setting up the callbacks.
+class CoverageCheckerConsumer : public ASTConsumer {
+public:
+  CoverageCheckerConsumer(CoverageChecker &Checker, Preprocessor &PP) {
+    // PP takes ownership.
+    PP.addPPCallbacks(llvm::make_unique<CoverageCheckerCallbacks>(Checker));
+  }
+};
+
+class CoverageCheckerAction : public SyntaxOnlyAction {
+public:
+  CoverageCheckerAction(CoverageChecker &Checker) : Checker(Checker) {}
+
+protected:
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+    StringRef InFile) override {
+    return llvm::make_unique<CoverageCheckerConsumer>(Checker,
+      CI.getPreprocessor());
+  }
+
+private:
+  CoverageChecker &Checker;
+};
+
+class CoverageCheckerFrontendActionFactory : public FrontendActionFactory {
+public:
+  CoverageCheckerFrontendActionFactory(CoverageChecker &Checker)
+    : Checker(Checker) {}
+
+  CoverageCheckerAction *create() override {
+    return new CoverageCheckerAction(Checker);
+  }
+
+private:
+  CoverageChecker &Checker;
+};
+
+// CoverageChecker class implementation.
+
+// Constructor.
+CoverageChecker::CoverageChecker(StringRef ModuleMapPath,
+    std::vector<std::string> &IncludePaths,
+    ArrayRef<std::string> CommandLine,
+    clang::ModuleMap *ModuleMap)
+  : ModuleMapPath(ModuleMapPath), IncludePaths(IncludePaths),
+    CommandLine(CommandLine),
+    ModMap(ModuleMap) {}
+
+// Create instance of CoverageChecker, to simplify setting up
+// subordinate objects.
+std::unique_ptr<CoverageChecker> CoverageChecker::createCoverageChecker(
+    StringRef ModuleMapPath, std::vector<std::string> &IncludePaths,
+    ArrayRef<std::string> CommandLine, clang::ModuleMap *ModuleMap) {
+
+  return llvm::make_unique<CoverageChecker>(ModuleMapPath, IncludePaths,
+                                            CommandLine, ModuleMap);
+}
+
+// Do checks.
+// Starting from the directory of the module.modulemap file,
+// Find all header files, optionally looking only at files
+// covered by the include path options, and compare against
+// the headers referenced by the module.modulemap file.
+// Display warnings for unaccounted-for header files.
+// Returns error_code of 0 if there were no errors or warnings, 1 if there
+//   were warnings, 2 if any other problem, such as if a bad
+//   module map path argument was specified.
+std::error_code CoverageChecker::doChecks() {
+  std::error_code returnValue;
+
+  // Collect the headers referenced in the modules.
+  collectModuleHeaders();
+
+  // Collect the file system headers.
+  if (!collectFileSystemHeaders())
+    return std::error_code(2, std::generic_category());
+
+  // Do the checks.  These save the problematic file names.
+  findUnaccountedForHeaders();
+
+  // Check for warnings.
+  if (!UnaccountedForHeaders.empty())
+    returnValue = std::error_code(1, std::generic_category());
+
+  return returnValue;
+}
+
+// The following functions are called by doChecks.
+
+// Collect module headers.
+// Walks the modules and collects referenced headers into
+// ModuleMapHeadersSet.
+void CoverageChecker::collectModuleHeaders() {
+  for (ModuleMap::module_iterator I = ModMap->module_begin(),
+    E = ModMap->module_end();
+    I != E; ++I) {
+    collectModuleHeaders(*I->second);
+  }
+}
+
+// Collect referenced headers from one module.
+// Collects the headers referenced in the given module into
+// ModuleMapHeadersSet.
+// FIXME: Doesn't collect files from umbrella header.
+bool CoverageChecker::collectModuleHeaders(const Module &Mod) {
+
+  if (const FileEntry *UmbrellaHeader = Mod.getUmbrellaHeader().Entry) {
+    // Collect umbrella header.
+    ModuleMapHeadersSet.insert(ModularizeUtilities::getCanonicalPath(
+      UmbrellaHeader->getName()));
+    // Preprocess umbrella header and collect the headers it references.
+    if (!collectUmbrellaHeaderHeaders(UmbrellaHeader->getName()))
+      return false;
+  }
+  else if (const DirectoryEntry *UmbrellaDir = Mod.getUmbrellaDir().Entry) {
+    // Collect headers in umbrella directory.
+    if (!collectUmbrellaHeaders(UmbrellaDir->getName()))
+      return false;
+  }
+
+  for (auto &HeaderKind : Mod.Headers)
+    for (auto &Header : HeaderKind)
+      ModuleMapHeadersSet.insert(ModularizeUtilities::getCanonicalPath(
+        Header.Entry->getName()));
+
+  for (auto MI = Mod.submodule_begin(), MIEnd = Mod.submodule_end();
+       MI != MIEnd; ++MI)
+    collectModuleHeaders(**MI);
+
+  return true;
+}
+
+// Collect headers from an umbrella directory.
+bool CoverageChecker::collectUmbrellaHeaders(StringRef UmbrellaDirName) {
+  // Initialize directory name.
+  SmallString<256> Directory(ModuleMapDirectory);
+  if (UmbrellaDirName.size())
+    sys::path::append(Directory, UmbrellaDirName);
+  if (Directory.size() == 0)
+    Directory = ".";
+  // Walk the directory.
+  std::error_code EC;
+  for (sys::fs::directory_iterator I(Directory.str(), EC), E; I != E;
+    I.increment(EC)) {
+    if (EC)
+      return false;
+    std::string File(I->path());
+    llvm::ErrorOr<sys::fs::basic_file_status> Status = I->status();
+    if (!Status)
+      return false;
+    sys::fs::file_type Type = Status->type();
+    // If the file is a directory, ignore the name and recurse.
+    if (Type == sys::fs::file_type::directory_file) {
+      if (!collectUmbrellaHeaders(File))
+        return false;
+      continue;
+    }
+    // If the file does not have a common header extension, ignore it.
+    if (!ModularizeUtilities::isHeader(File))
+      continue;
+    // Save header name.
+    ModuleMapHeadersSet.insert(ModularizeUtilities::getCanonicalPath(File));
+  }
+  return true;
+}
+
+// Collect headers rferenced from an umbrella file.
+bool
+CoverageChecker::collectUmbrellaHeaderHeaders(StringRef UmbrellaHeaderName) {
+
+  SmallString<256> PathBuf(ModuleMapDirectory);
+
+  // If directory is empty, it's the current directory.
+  if (ModuleMapDirectory.length() == 0)
+    sys::fs::current_path(PathBuf);
+
+  // Create the compilation database.
+  std::unique_ptr<CompilationDatabase> Compilations;
+  Compilations.reset(new FixedCompilationDatabase(Twine(PathBuf), CommandLine));
+
+  std::vector<std::string> HeaderPath;
+  HeaderPath.push_back(UmbrellaHeaderName);
+
+  // Create the tool and run the compilation.
+  ClangTool Tool(*Compilations, HeaderPath);
+  int HadErrors = Tool.run(new CoverageCheckerFrontendActionFactory(*this));
+
+  // If we had errors, exit early.
+  return !HadErrors;
+}
+
+// Called from CoverageCheckerCallbacks to track a header included
+// from an umbrella header.
+void CoverageChecker::collectUmbrellaHeaderHeader(StringRef HeaderName) {
+
+  SmallString<256> PathBuf(ModuleMapDirectory);
+  // If directory is empty, it's the current directory.
+  if (ModuleMapDirectory.length() == 0)
+    sys::fs::current_path(PathBuf);
+  // HeaderName will have an absolute path, so if it's the module map
+  // directory, we remove it, also skipping trailing separator.
+  if (HeaderName.startswith(PathBuf))
+    HeaderName = HeaderName.substr(PathBuf.size() + 1);
+  // Save header name.
+  ModuleMapHeadersSet.insert(ModularizeUtilities::getCanonicalPath(HeaderName));
+}
+
+// Collect file system header files.
+// This function scans the file system for header files,
+// starting at the directory of the module.modulemap file,
+// optionally filtering out all but the files covered by
+// the include path options.
+// Returns true if no errors.
+bool CoverageChecker::collectFileSystemHeaders() {
+
+  // Get directory containing the module.modulemap file.
+  // Might be relative to current directory, absolute, or empty.
+  ModuleMapDirectory = ModularizeUtilities::getDirectoryFromPath(ModuleMapPath);
+
+  // If no include paths specified, we do the whole tree starting
+  // at the module.modulemap directory.
+  if (IncludePaths.size() == 0) {
+    if (!collectFileSystemHeaders(StringRef("")))
+      return false;
+  }
+  else {
+    // Otherwise we only look at the sub-trees specified by the
+    // include paths.
+    for (std::vector<std::string>::const_iterator I = IncludePaths.begin(),
+      E = IncludePaths.end();
+      I != E; ++I) {
+      if (!collectFileSystemHeaders(*I))
+        return false;
+    }
+  }
+
+  // Sort it, because different file systems might order the file differently.
+  std::sort(FileSystemHeaders.begin(), FileSystemHeaders.end());
+
+  return true;
+}
+
+// Collect file system header files from the given path.
+// This function scans the file system for header files,
+// starting at the given directory, which is assumed to be
+// relative to the directory of the module.modulemap file.
+// \returns True if no errors.
+bool CoverageChecker::collectFileSystemHeaders(StringRef IncludePath) {
+
+  // Initialize directory name.
+  SmallString<256> Directory(ModuleMapDirectory);
+  if (IncludePath.size())
+    sys::path::append(Directory, IncludePath);
+  if (Directory.size() == 0)
+    Directory = ".";
+  if (IncludePath.startswith("/") || IncludePath.startswith("\\") ||
+    ((IncludePath.size() >= 2) && (IncludePath[1] == ':'))) {
+    llvm::errs() << "error: Include path \"" << IncludePath
+      << "\" is not relative to the module map file.\n";
+    return false;
+  }
+
+  // Recursively walk the directory tree.
+  std::error_code EC;
+  int Count = 0;
+  for (sys::fs::recursive_directory_iterator I(Directory.str(), EC), E; I != E;
+    I.increment(EC)) {
+    if (EC)
+      return false;
+    //std::string file(I->path());
+    StringRef file(I->path());
+    llvm::ErrorOr<sys::fs::basic_file_status> Status = I->status();
+    if (!Status)
+      return false;
+    sys::fs::file_type type = Status->type();
+    // If the file is a directory, ignore the name (but still recurses).
+    if (type == sys::fs::file_type::directory_file)
+      continue;
+    // Assume directories or files starting with '.' are private and not to
+    // be considered.
+    if ((file.find("\\.") != StringRef::npos) ||
+        (file.find("/.") != StringRef::npos))
+      continue;
+    // If the file does not have a common header extension, ignore it.
+    if (!ModularizeUtilities::isHeader(file))
+      continue;
+    // Save header name.
+    FileSystemHeaders.push_back(ModularizeUtilities::getCanonicalPath(file));
+    Count++;
+  }
+  if (Count == 0) {
+    llvm::errs() << "warning: No headers found in include path: \""
+      << IncludePath << "\"\n";
+  }
+  return true;
+}
+
+// Find headers unaccounted-for in module map.
+// This function compares the list of collected header files
+// against those referenced in the module map.  Display
+// warnings for unaccounted-for header files.
+// Save unaccounted-for file list for possible.
+// fixing action.
+// FIXME: There probably needs to be some canonalization
+// of file names so that header path can be correctly
+// matched.  Also, a map could be used for the headers
+// referenced in the module, but
+void CoverageChecker::findUnaccountedForHeaders() {
+  // Walk over file system headers.
+  for (std::vector<std::string>::const_iterator I = FileSystemHeaders.begin(),
+    E = FileSystemHeaders.end();
+    I != E; ++I) {
+    // Look for header in module map.
+    if (ModuleMapHeadersSet.insert(*I).second) {
+      UnaccountedForHeaders.push_back(*I);
+      llvm::errs() << "warning: " << ModuleMapPath
+        << " does not account for file: " << *I << "\n";
+    }
+  }
+}

external/llvm-project/clang-tools-extra/modularize/CoverageChecker.h

@@ -0,0 +1,164 @@
+//===-- CoverageChecker.h - Module map coverage checker -*- C++ -*-------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--------------------------------------------------------------------===//
+///
+/// \file
+/// \brief Definitions for CoverageChecker.
+///
+//===--------------------------------------------------------------------===//
+
+#ifndef COVERAGECHECKER_H
+#define COVERAGECHECKER_H
+
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/ModuleMap.h"
+#include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Host.h"
+#include <string>
+#include <vector>
+
+namespace Modularize {
+
+/// Module map checker class.
+/// This is the heart of the checker.
+/// The doChecks function does the main work.
+/// The data members store the options and internally collected data.
+class CoverageChecker {
+  // Checker arguments.
+
+  /// The module.modulemap file path. Can be relative or absolute.
+  llvm::StringRef ModuleMapPath;
+  /// The include paths to check for files.
+  /// (Note that other directories above these paths are ignored.
+  /// To expect all files to be accounted for from the module.modulemap
+  /// file directory on down, leave this empty.)
+  std::vector<std::string> IncludePaths;
+  /// The remaining arguments, to be passed to the front end.
+  llvm::ArrayRef<std::string> CommandLine;
+  /// The module map.
+  clang::ModuleMap *ModMap;
+
+  // Internal data.
+
+  /// Directory containing the module map.
+  /// Might be relative to the current directory, or absolute.
+  std::string ModuleMapDirectory;
+  /// Set of all the headers found in the module map.
+  llvm::StringSet<llvm::MallocAllocator> ModuleMapHeadersSet;
+  /// All the headers found in the file system starting at the
+  /// module map, or the union of those from the include paths.
+  std::vector<std::string> FileSystemHeaders;
+  /// Headers found in file system, but not in module map.
+  std::vector<std::string> UnaccountedForHeaders;
+
+public:
+  /// Constructor.
+  /// You can use the static createCoverageChecker to create an instance
+  /// of this object.
+  /// \param ModuleMapPath The module.modulemap file path.
+  ///   Can be relative or absolute.
+  /// \param IncludePaths The include paths to check for files.
+  ///   (Note that other directories above these paths are ignored.
+  ///   To expect all files to be accounted for from the module.modulemap
+  ///   file directory on down, leave this empty.)
+  /// \param CommandLine Compile command line arguments.
+  /// \param ModuleMap The module map to check.
+  CoverageChecker(llvm::StringRef ModuleMapPath,
+    std::vector<std::string> &IncludePaths,
+    llvm::ArrayRef<std::string> CommandLine,
+    clang::ModuleMap *ModuleMap);
+
+  /// Create instance of CoverageChecker.
+  /// \param ModuleMapPath The module.modulemap file path.
+  ///   Can be relative or absolute.
+  /// \param IncludePaths The include paths to check for files.
+  ///   (Note that other directories above these paths are ignored.
+  ///   To expect all files to be accounted for from the module.modulemap
+  ///   file directory on down, leave this empty.)
+  /// \param CommandLine Compile command line arguments.
+  /// \param ModuleMap The module map to check.
+  /// \returns Initialized CoverageChecker object.
+  static std::unique_ptr<CoverageChecker> createCoverageChecker(
+      llvm::StringRef ModuleMapPath, std::vector<std::string> &IncludePaths,
+      llvm::ArrayRef<std::string> CommandLine, clang::ModuleMap *ModuleMap);
+
+  /// Do checks.
+  /// Starting from the directory of the module.modulemap file,
+  /// Find all header files, optionally looking only at files
+  /// covered by the include path options, and compare against
+  /// the headers referenced by the module.modulemap file.
+  /// Display warnings for unaccounted-for header files.
+  /// \returns 0 if there were no errors or warnings, 1 if there
+  ///   were warnings, 2 if any other problem, such as a bad
+  ///   module map path argument was specified.
+  std::error_code doChecks();
+
+  // The following functions are called by doChecks.
+
+  /// Collect module headers.
+  /// Walks the modules and collects referenced headers into
+  /// ModuleMapHeadersSet.
+  void collectModuleHeaders();
+
+  /// Collect referenced headers from one module.
+  /// Collects the headers referenced in the given module into
+  /// ModuleMapHeadersSet.
+  /// \param Mod The module reference.
+  /// \return True if no errors.
+  bool collectModuleHeaders(const clang::Module &Mod);
+
+  /// Collect headers from an umbrella directory.
+  /// \param UmbrellaDirName The umbrella directory name.
+  /// \return True if no errors.
+  bool collectUmbrellaHeaders(llvm::StringRef UmbrellaDirName);
+
+  /// Collect headers rferenced from an umbrella file.
+  /// \param UmbrellaHeaderName The umbrella file path.
+  /// \return True if no errors.
+  bool collectUmbrellaHeaderHeaders(llvm::StringRef UmbrellaHeaderName);
+
+  /// Called from CoverageCheckerCallbacks to track a header included
+  /// from an umbrella header.
+  /// \param HeaderName The header file path.
+  void collectUmbrellaHeaderHeader(llvm::StringRef HeaderName);
+
+  /// Collect file system header files.
+  /// This function scans the file system for header files,
+  /// starting at the directory of the module.modulemap file,
+  /// optionally filtering out all but the files covered by
+  /// the include path options.
+  /// \returns True if no errors.
+  bool collectFileSystemHeaders();
+
+  /// Collect file system header files from the given path.
+  /// This function scans the file system for header files,
+  /// starting at the given directory, which is assumed to be
+  /// relative to the directory of the module.modulemap file.
+  /// \returns True if no errors.
+  bool collectFileSystemHeaders(llvm::StringRef IncludePath);
+
+  /// Find headers unaccounted-for in module map.
+  /// This function compares the list of collected header files
+  /// against those referenced in the module map.  Display
+  /// warnings for unaccounted-for header files.
+  /// Save unaccounted-for file list for possible.
+  /// fixing action.
+  void findUnaccountedForHeaders();
+};
+
+} // end namespace Modularize
+
+#endif // COVERAGECHECKER_H

external/llvm-project/clang-tools-extra/modularize/Modularize.h

@@ -0,0 +1,54 @@
+//===--- Modularize.h - Common definitions for Modularize -*- C++ -*-----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--------------------------------------------------------------------===//
+///
+/// \file
+/// \brief Common definitions for Modularize.
+///
+//===--------------------------------------------------------------------===//
+
+#ifndef MODULARIZE_H
+#define MODULARIZE_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include <string>
+#include <vector>
+
+// Save the program name for error messages.
+extern const char *Argv0;
+// Save the command line for comments.
+extern std::string CommandLine;
+
+// Dependency types.
+typedef llvm::SmallVector<std::string, 4> DependentsVector;
+typedef llvm::StringMap<DependentsVector> DependencyMap;
+
+// Global function declarations.
+
+/// Create the module map file.
+/// \param ModuleMapPath The path to the module map file to be generated.
+/// \param HeaderFileNames The list of header files, absolute native paths.
+/// \param ProblemFileNames The list of problem header files.
+/// \param Dependencies Map of headers that depend on other headers.
+/// \param HeaderPrefix Tells the code where the headers are, if they
+///   aren's in the current directory, allowing the generator to strip
+///   the leading, non-relative beginning of the header paths.
+/// \brief RootModuleName If not empty, specifies that a root module
+///   should be created with this name.
+/// \returns True if successful.
+bool createModuleMap(llvm::StringRef ModuleMapPath,
+                     llvm::ArrayRef<std::string> HeaderFileNames,
+                     llvm::ArrayRef<std::string> ProblemFileNames,
+                     DependencyMap &Dependencies, llvm::StringRef HeaderPrefix,
+                     llvm::StringRef RootModuleName);
+
+#endif // MODULARIZE_H

external/llvm-project/clang-tools-extra/modularize/ModularizeUtilities.h

@@ -0,0 +1,225 @@
+//=====-- ModularizeUtilities.h - Utilities for modularize -*- C++ -*-======//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--------------------------------------------------------------------===//
+///
+/// \file
+/// \brief ModularizeUtilities class definition.
+///
+//===--------------------------------------------------------------------===//
+
+#ifndef MODULARIZEUTILITIES_H
+#define MODULARIZEUTILITIES_H
+
+#include "Modularize.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/ModuleMap.h"
+#include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringSet.h"
+#include <string>
+#include <vector>
+
+namespace Modularize {
+
+/// Modularize utilities class.
+/// Support functions and data for modularize.
+class ModularizeUtilities {
+public:
+  // Input arguments.
+
+  /// The input file paths.
+  std::vector<std::string> InputFilePaths;
+  /// The header prefix.
+  llvm::StringRef HeaderPrefix;
+  /// The path of problem files list file.
+  llvm::StringRef ProblemFilesPath;
+
+  // Output data.
+
+  /// List of top-level header files.
+  llvm::SmallVector<std::string, 32> HeaderFileNames;
+  /// Map of top-level header file dependencies.
+  DependencyMap Dependencies;
+  /// True if we have module maps.
+  bool HasModuleMap;
+  /// Missing header count.
+  int MissingHeaderCount;
+  /// List of header files with no problems during the first pass,
+  /// that is, no compile errors.
+  llvm::SmallVector<std::string, 32> GoodFileNames;
+  /// List of header files with problems.
+  llvm::SmallVector<std::string, 32> ProblemFileNames;
+
+  // Functions.
+
+  /// Constructor.
+  /// You can use the static createModularizeUtilities to create an instance
+  /// of this object.
+  /// \param InputPaths The input file paths.
+  /// \param Prefix The headear path prefix.
+  /// \param ProblemFilesListPath The problem header list path.
+  ModularizeUtilities(std::vector<std::string> &InputPaths,
+                      llvm::StringRef Prefix,
+                      llvm::StringRef ProblemFilesListPath);
+
+  /// Create instance of ModularizeUtilities.
+  /// \param InputPaths The input file paths.
+  /// \param Prefix The headear path prefix.
+  /// \param ProblemFilesListPath The problem header list path.
+  /// \returns Initialized ModularizeUtilities object.
+  static ModularizeUtilities *createModularizeUtilities(
+      std::vector<std::string> &InputPaths,
+      llvm::StringRef Prefix,
+      llvm::StringRef ProblemFilesListPath);
+
+  /// Load header list and dependencies.
+  /// \returns std::error_code.
+  std::error_code loadAllHeaderListsAndDependencies();
+
+  /// Do coverage checks.
+  /// For each loaded module map, do header coverage check.
+  /// Starting from the directory of the module.map file,
+  /// Find all header files, optionally looking only at files
+  /// covered by the include path options, and compare against
+  /// the headers referenced by the module.map file.
+  /// Display warnings for unaccounted-for header files.
+  /// \param IncludePaths The include paths to check for files.
+  ///   (Note that other directories above these paths are ignored.
+  ///   To expect all files to be accounted for from the module.modulemap
+  ///   file directory on down, leave this empty.)
+  /// \param CommandLine Compile command line arguments.
+  /// \returns 0 if there were no errors or warnings, 1 if there
+  ///   were warnings, 2 if any other problem, such as a bad
+  ///   module map path argument was specified.
+  std::error_code doCoverageCheck(std::vector<std::string> &IncludePaths,
+                                  llvm::ArrayRef<std::string> CommandLine);
+
+  /// Add unique problem file.
+  /// Also standardizes the path.
+  /// \param FilePath Problem file path.
+  void addUniqueProblemFile(std::string FilePath);
+
+  /// Add file with no compile errors.
+  /// Also standardizes the path.
+  /// \param FilePath Problem file path.
+  void addNoCompileErrorsFile(std::string FilePath);
+
+  /// List problem files.
+  void displayProblemFiles();
+
+  /// List files with no problems.
+  void displayGoodFiles();
+
+  /// List files with problem files commented out.
+  void displayCombinedFiles();
+
+  // Internal.
+
+protected:
+
+  /// Load single header list and dependencies.
+  /// \param InputPath The input file path.
+  /// \returns std::error_code.
+  std::error_code loadSingleHeaderListsAndDependencies(
+      llvm::StringRef InputPath);
+
+  /// Load problem header list.
+  /// \param InputPath The input file path.
+  /// \returns std::error_code.
+  std::error_code loadProblemHeaderList(
+    llvm::StringRef InputPath);
+
+  /// Load single module map and extract header file list.
+  /// \param InputPath The input file path.
+  /// \returns std::error_code.
+  std::error_code loadModuleMap(
+    llvm::StringRef InputPath);
+
+  /// Collect module Map headers.
+  /// Walks the modules and collects referenced headers into
+  /// HeaderFileNames.
+  /// \param ModMap A loaded module map object.
+  /// \return True if no errors.
+  bool collectModuleMapHeaders(clang::ModuleMap *ModMap);
+
+  /// Collect referenced headers from one module.
+  /// Collects the headers referenced in the given module into
+  /// HeaderFileNames.
+  /// \param Mod The module reference.
+  /// \return True if no errors.
+  bool collectModuleHeaders(const clang::Module &Mod);
+
+  /// Collect headers from an umbrella directory.
+  /// \param UmbrellaDirName The umbrella directory name.
+  /// \return True if no errors.
+  bool collectUmbrellaHeaders(llvm::StringRef UmbrellaDirName,
+    DependentsVector &Dependents);
+
+public:
+
+  // Utility functions.
+
+  /// Convert header path to canonical form.
+  /// The canonical form is basically just use forward slashes,
+  /// and remove "./".
+  /// \param FilePath The file path.
+  /// \returns The file path in canonical form.
+  static std::string getCanonicalPath(llvm::StringRef FilePath);
+
+  /// Check for header file extension.
+  /// If the file extension is .h, .inc, or missing, it's
+  /// assumed to be a header.
+  /// \param FileName The file name.  Must not be a directory.
+  /// \returns true if it has a header extension or no extension.
+  static bool isHeader(llvm::StringRef FileName);
+
+  /// Get directory path component from file path.
+  /// \returns the component of the given path, which will be
+  /// relative if the given path is relative, absolute if the
+  /// given path is absolute, or "." if the path has no leading
+  /// path component.
+  static std::string getDirectoryFromPath(llvm::StringRef Path);
+
+  // Internal data.
+
+  /// Options controlling the language variant.
+  std::shared_ptr<clang::LangOptions> LangOpts;
+  /// Diagnostic IDs.
+  const llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagIDs;
+  /// Options controlling the diagnostic engine.
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> DiagnosticOpts;
+  /// Diagnostic consumer.
+  clang::TextDiagnosticPrinter DC;
+  /// Diagnostic engine.
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> Diagnostics;
+  /// Options controlling the target.
+  std::shared_ptr<clang::TargetOptions> TargetOpts;
+  /// Target information.
+  llvm::IntrusiveRefCntPtr<clang::TargetInfo> Target;
+  /// Options controlling the file system manager.
+  clang::FileSystemOptions FileSystemOpts;
+  /// File system manager.
+  llvm::IntrusiveRefCntPtr<clang::FileManager> FileMgr;
+  /// Source manager.
+  llvm::IntrusiveRefCntPtr<clang::SourceManager> SourceMgr;
+  /// Header search manager.
+  std::unique_ptr<clang::HeaderSearch> HeaderInfo;
+  // The loaded module map objects.
+  std::vector<std::unique_ptr<clang::ModuleMap>> ModuleMaps;
+};
+
+} // end namespace Modularize
+
+#endif // MODULARIZEUTILITIES_H

external/llvm-project/clang-tools-extra/modularize/ModuleAssistant.cpp

@@ -0,0 +1,316 @@
+//===--- ModuleAssistant.cpp - Module map generation manager -*- C++ -*---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file defines the module generation entry point function,
+// createModuleMap, a Module class for representing a module,
+// and various implementation functions for doing the underlying
+// work, described below.
+//
+// The "Module" class represents a module, with members for storing the module
+// name, associated header file names, and sub-modules, and an "output"
+// function that recursively writes the module definitions.
+//
+// The "createModuleMap" function implements the top-level logic of the
+// assistant mode.  It calls a loadModuleDescriptions function to walk
+// the header list passed to it and creates a tree of Module objects
+// representing the module hierarchy, represented by a "Module" object,
+// the "RootModule".  This root module may or may not represent an actual
+// module in the module map, depending on the "--root-module" option passed
+// to modularize.  It then calls a writeModuleMap function to set up the
+// module map file output and walk the module tree, outputting the module
+// map file using a stream obtained and managed by an
+// llvm::ToolOutputFile object.
+//
+//===---------------------------------------------------------------------===//
+
+#include "Modularize.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include <vector>
+
+// Local definitions:
+
+namespace {
+
+// Internal class definitions:
+
+// Represents a module.
+class Module {
+public:
+  Module(llvm::StringRef Name, bool Problem);
+  Module();
+  ~Module();
+  bool output(llvm::raw_fd_ostream &OS, int Indent);
+  Module *findSubModule(llvm::StringRef SubName);
+
+public:
+  std::string Name;
+  std::vector<std::string> HeaderFileNames;
+  std::vector<Module *> SubModules;
+  bool IsProblem;
+};
+
+} // end anonymous namespace.
+
+// Module functions:
+
+// Constructors.
+Module::Module(llvm::StringRef Name, bool Problem)
+  : Name(Name), IsProblem(Problem) {}
+Module::Module() : IsProblem(false) {}
+
+// Destructor.
+Module::~Module() {
+  // Free submodules.
+  while (!SubModules.empty()) {
+    Module *last = SubModules.back();
+    SubModules.pop_back();
+    delete last;
+  }
+}
+
+// Write a module hierarchy to the given output stream.
+bool Module::output(llvm::raw_fd_ostream &OS, int Indent) {
+  // If this is not the nameless root module, start a module definition.
+  if (Name.size() != 0) {
+    OS.indent(Indent);
+    OS << "module " << Name << " {\n";
+    Indent += 2;
+  }
+
+  // Output submodules.
+  for (auto I = SubModules.begin(), E = SubModules.end(); I != E; ++I) {
+    if (!(*I)->output(OS, Indent))
+      return false;
+  }
+
+  // Output header files.
+  for (auto I = HeaderFileNames.begin(), E = HeaderFileNames.end(); I != E;
+       ++I) {
+    OS.indent(Indent);
+    if (IsProblem || strstr((*I).c_str(), ".inl"))
+      OS << "exclude header \"" << *I << "\"\n";
+    else
+      OS << "header \"" << *I << "\"\n";
+  }
+
+  // If this module has header files, output export directive.
+  if (HeaderFileNames.size() != 0) {
+    OS.indent(Indent);
+    OS << "export *\n";
+  }
+
+  // If this is not the nameless root module, close the module definition.
+  if (Name.size() != 0) {
+    Indent -= 2;
+    OS.indent(Indent);
+    OS << "}\n";
+  }
+
+  return true;
+}
+
+// Lookup a sub-module.
+Module *Module::findSubModule(llvm::StringRef SubName) {
+  for (auto I = SubModules.begin(), E = SubModules.end(); I != E; ++I) {
+    if ((*I)->Name == SubName)
+      return *I;
+  }
+  return nullptr;
+}
+
+// Implementation functions:
+
+// Reserved keywords in module.modulemap syntax.
+// Keep in sync with keywords in module map parser in Lex/ModuleMap.cpp,
+// such as in ModuleMapParser::consumeToken().
+static const char *const ReservedNames[] = {
+  "config_macros", "export",   "module", "conflict", "framework",
+  "requires",      "exclude",  "header", "private",  "explicit",
+  "link",          "umbrella", "extern", "use",      nullptr // Flag end.
+};
+
+// Convert module name to a non-keyword.
+// Prepends a '_' to the name if and only if the name is a keyword.
+static std::string
+ensureNoCollisionWithReservedName(llvm::StringRef MightBeReservedName) {
+  std::string SafeName = MightBeReservedName;
+  for (int Index = 0; ReservedNames[Index] != nullptr; ++Index) {
+    if (MightBeReservedName == ReservedNames[Index]) {
+      SafeName.insert(0, "_");
+      break;
+    }
+  }
+  return SafeName;
+}
+
+// Convert module name to a non-keyword.
+// Prepends a '_' to the name if and only if the name is a keyword.
+static std::string
+ensureVaidModuleName(llvm::StringRef MightBeInvalidName) {
+  std::string SafeName = MightBeInvalidName;
+  std::replace(SafeName.begin(), SafeName.end(), '-', '_');
+  std::replace(SafeName.begin(), SafeName.end(), '.', '_');
+  if (isdigit(SafeName[0]))
+    SafeName = "_" + SafeName;
+  return SafeName;
+}
+
+// Add one module, given a header file path.
+static bool addModuleDescription(Module *RootModule,
+                                 llvm::StringRef HeaderFilePath,
+                                 llvm::StringRef HeaderPrefix,
+                                 DependencyMap &Dependencies,
+                                 bool IsProblemFile) {
+  Module *CurrentModule = RootModule;
+  DependentsVector &FileDependents = Dependencies[HeaderFilePath];
+  std::string FilePath;
+  // Strip prefix.
+  // HeaderFilePath should be compared to natively-canonicalized Prefix.
+  llvm::SmallString<256> NativePath, NativePrefix;
+  llvm::sys::path::native(HeaderFilePath, NativePath);
+  llvm::sys::path::native(HeaderPrefix, NativePrefix);
+  if (NativePath.startswith(NativePrefix))
+    FilePath = NativePath.substr(NativePrefix.size() + 1);
+  else
+    FilePath = HeaderFilePath;
+  int Count = FileDependents.size();
+  // Headers that go into modules must not depend on other files being
+  // included first.  If there are any dependents, warn user and omit.
+  if (Count != 0) {
+    llvm::errs() << "warning: " << FilePath
+                 << " depends on other headers being included first,"
+                    " meaning the module.modulemap won't compile."
+                    "  This header will be omitted from the module map.\n";
+    return true;
+  }
+  // Make canonical.
+  std::replace(FilePath.begin(), FilePath.end(), '\\', '/');
+  // Insert module into tree, using subdirectories as submodules.
+  for (llvm::sys::path::const_iterator I = llvm::sys::path::begin(FilePath),
+                                       E = llvm::sys::path::end(FilePath);
+       I != E; ++I) {
+    if ((*I)[0] == '.')
+      continue;
+    std::string Stem = llvm::sys::path::stem(*I);
+    Stem = ensureNoCollisionWithReservedName(Stem);
+    Stem = ensureVaidModuleName(Stem);
+    Module *SubModule = CurrentModule->findSubModule(Stem);
+    if (!SubModule) {
+      SubModule = new Module(Stem, IsProblemFile);
+      CurrentModule->SubModules.push_back(SubModule);
+    }
+    CurrentModule = SubModule;
+  }
+  // Add header file name to headers.
+  CurrentModule->HeaderFileNames.push_back(FilePath);
+  return true;
+}
+
+// Create the internal module tree representation.
+static Module *loadModuleDescriptions(
+    llvm::StringRef RootModuleName, llvm::ArrayRef<std::string> HeaderFileNames,
+    llvm::ArrayRef<std::string> ProblemFileNames,
+    DependencyMap &Dependencies, llvm::StringRef HeaderPrefix) {
+
+  // Create root module.
+  auto *RootModule = new Module(RootModuleName, false);
+
+  llvm::SmallString<256> CurrentDirectory;
+  llvm::sys::fs::current_path(CurrentDirectory);
+
+  // If no header prefix, use current directory.
+  if (HeaderPrefix.size() == 0)
+    HeaderPrefix = CurrentDirectory;
+
+  // Walk the header file names and output the module map.
+  for (llvm::ArrayRef<std::string>::iterator I = HeaderFileNames.begin(),
+                                             E = HeaderFileNames.end();
+       I != E; ++I) {
+    std::string Header(*I);
+    bool IsProblemFile = false;
+    for (auto &ProblemFile : ProblemFileNames) {
+      if (ProblemFile == Header) {
+        IsProblemFile = true;
+        break;
+      }
+    }
+    // Add as a module.
+    if (!addModuleDescription(RootModule, Header, HeaderPrefix, Dependencies, IsProblemFile))
+      return nullptr;
+  }
+
+  return RootModule;
+}
+
+// Kick off the writing of the module map.
+static bool writeModuleMap(llvm::StringRef ModuleMapPath,
+                           llvm::StringRef HeaderPrefix, Module *RootModule) {
+  llvm::SmallString<256> HeaderDirectory(ModuleMapPath);
+  llvm::sys::path::remove_filename(HeaderDirectory);
+  llvm::SmallString<256> FilePath;
+
+  // Get the module map file path to be used.
+  if ((HeaderDirectory.size() == 0) && (HeaderPrefix.size() != 0)) {
+    FilePath = HeaderPrefix;
+    // Prepend header file name prefix if it's not absolute.
+    llvm::sys::path::append(FilePath, ModuleMapPath);
+    llvm::sys::path::native(FilePath);
+  } else {
+    FilePath = ModuleMapPath;
+    llvm::sys::path::native(FilePath);
+  }
+
+  // Set up module map output file.
+  std::error_code EC;
+  llvm::ToolOutputFile Out(FilePath, EC, llvm::sys::fs::F_Text);
+  if (EC) {
+    llvm::errs() << Argv0 << ": error opening " << FilePath << ":"
+                 << EC.message() << "\n";
+    return false;
+  }
+
+  // Get output stream from tool output buffer/manager.
+  llvm::raw_fd_ostream &OS = Out.os();
+
+  // Output file comment.
+  OS << "// " << ModuleMapPath << "\n";
+  OS << "// Generated by: " << CommandLine << "\n\n";
+
+  // Write module hierarchy from internal representation.
+  if (!RootModule->output(OS, 0))
+    return false;
+
+  // Tell ToolOutputFile that we want to keep the file.
+  Out.keep();
+
+  return true;
+}
+
+// Global functions:
+
+// Module map generation entry point.
+bool createModuleMap(llvm::StringRef ModuleMapPath,
+                     llvm::ArrayRef<std::string> HeaderFileNames,
+                     llvm::ArrayRef<std::string> ProblemFileNames,
+                     DependencyMap &Dependencies, llvm::StringRef HeaderPrefix,
+                     llvm::StringRef RootModuleName) {
+  // Load internal representation of modules.
+  std::unique_ptr<Module> RootModule(
+    loadModuleDescriptions(
+      RootModuleName, HeaderFileNames, ProblemFileNames, Dependencies,
+      HeaderPrefix));
+  if (!RootModule.get())
+    return false;
+
+  // Write module map file.
+  return writeModuleMap(ModuleMapPath, HeaderPrefix, RootModule.get());
+}

external/llvm-project/clang-tools-extra/modularize/PreprocessorTracker.h

@@ -0,0 +1,87 @@
+//===- PreprocessorTracker.h - Tracks preprocessor activities -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--------------------------------------------------------------------===//
+///
+/// \file
+/// \brief Macro expansions and preprocessor conditional consistency checker.
+///
+//===--------------------------------------------------------------------===//
+
+#ifndef MODULARIZE_PREPROCESSOR_TRACKER_H
+#define MODULARIZE_PREPROCESSOR_TRACKER_H
+
+#include "clang/Lex/Preprocessor.h"
+
+namespace Modularize {
+
+/// \brief Preprocessor tracker for modularize.
+///
+/// The PreprocessorTracker class defines an API for
+/// checking macro expansions and preprocessor conditional expressions
+/// in a header file for consistency among one or more compilations of
+/// the header in a #include scenario.  This is for helping a user
+/// find which macro expansions or conditionals might be problematic with
+/// respect to using the headers in the modules scenario, because they
+/// evaluate to different values depending on how or where a header
+/// is included.
+///
+/// The handlePreprocessorEntry function implementation will register
+/// a PPCallbacks object in the given Preprocessor object.  The calls to
+/// the callbacks will collect information about the macro expansions
+/// and preprocessor conditionals encountered, for later analysis and
+/// reporting of inconsistencies between runs performed by calls to
+/// the reportInconsistentMacros and reportInconsistentConditionals
+/// functions respectively.  The handlePreprocessorExit informs the
+/// implementation that a preprocessing session is complete, allowing
+/// it to do any needed compilation completion activities in the checker.
+class PreprocessorTracker {
+public:
+  virtual ~PreprocessorTracker();
+
+  // Handle entering a preprocessing session.
+  // (Called after a Preprocessor object is created, but before preprocessing.)
+  virtual void handlePreprocessorEntry(clang::Preprocessor &PP,
+                                       llvm::StringRef RootHeaderFile) = 0;
+  // Handle exiting a preprocessing session.
+  // (Called after preprocessing is complete, but before the Preprocessor
+  // object is destroyed.)
+  virtual void handlePreprocessorExit() = 0;
+
+  // Handle include directive.
+  // This function is called every time an include directive is seen by the
+  // preprocessor, for the purpose of later checking for 'extern "" {}' or
+  // "namespace {}" blocks containing #include directives.
+  virtual void handleIncludeDirective(llvm::StringRef DirectivePath,
+                                      int DirectiveLine, int DirectiveColumn,
+                                      llvm::StringRef TargetPath) = 0;
+
+  // Check for include directives within the given source line range.
+  // Report errors if any found.  Returns true if no include directives
+  // found in block.
+  virtual bool checkForIncludesInBlock(clang::Preprocessor &PP,
+                                       clang::SourceRange BlockSourceRange,
+                                       const char *BlockIdentifierMessage,
+                                       llvm::raw_ostream &OS) = 0;
+
+  // Report on inconsistent macro instances.
+  // Returns true if any mismatches.
+  virtual bool reportInconsistentMacros(llvm::raw_ostream &OS) = 0;
+
+  // Report on inconsistent conditional directive instances.
+  // Returns true if any mismatches.
+  virtual bool reportInconsistentConditionals(llvm::raw_ostream &OS) = 0;
+
+  // Create instance of PreprocessorTracker.
+  static PreprocessorTracker *create(
+    llvm::SmallVector<std::string, 32> &Headers,
+    bool DoBlockCheckHeaderListOnly);
+};
+
+} // end namespace Modularize
+
+#endif