vendor/llvm-project/llvmorg-10-init-17466-ge26a78e7085

5 files changed, 317 insertions, 29 deletions

diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 7436c7256327..b4d5a29ca695 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -122,6 +122,32 @@ DependencyScanningFilesystemSharedCache::get(StringRef Key) {
   return It.first->getValue();
 }
 
+/// Whitelist file extensions that should be minimized, treating no extension as
+/// a source file that should be minimized.
+///
+/// This is kinda hacky, it would be better if we knew what kind of file Clang
+/// was expecting instead.
+static bool shouldMinimize(StringRef Filename) {
+  StringRef Ext = llvm::sys::path::extension(Filename);
+  if (Ext.empty())
+    return true; // C++ standard library
+  return llvm::StringSwitch<bool>(Ext)
+    .CasesLower(".c", ".cc", ".cpp", ".c++", ".cxx", true)
+    .CasesLower(".h", ".hh", ".hpp", ".h++", ".hxx", true)
+    .CasesLower(".m", ".mm", true)
+    .CasesLower(".i", ".ii", ".mi", ".mmi", true)
+    .CasesLower(".def", ".inc", true)
+    .Default(false);
+}
+
+
+static bool shouldCacheStatFailures(StringRef Filename) {
+  StringRef Ext = llvm::sys::path::extension(Filename);
+  if (Ext.empty())
+    return false; // This may be the module cache directory.
+  return shouldMinimize(Filename); // Only cache stat failures on source files.
+}
+
 llvm::ErrorOr<const CachedFileSystemEntry *>
 DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
     const StringRef Filename) {
@@ -132,7 +158,8 @@ DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
   // FIXME: Handle PCM/PCH files.
   // FIXME: Handle module map files.
 
-  bool KeepOriginalSource = IgnoredFiles.count(Filename);
+  bool KeepOriginalSource = IgnoredFiles.count(Filename) ||
+                            !shouldMinimize(Filename);
   DependencyScanningFilesystemSharedCache::SharedFileSystemEntry
       &SharedCacheEntry = SharedCache.get(Filename);
   const CachedFileSystemEntry *Result;
@@ -143,9 +170,16 @@ DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
     if (!CacheEntry.isValid()) {
       llvm::vfs::FileSystem &FS = getUnderlyingFS();
       auto MaybeStatus = FS.status(Filename);
-      if (!MaybeStatus)
-        CacheEntry = CachedFileSystemEntry(MaybeStatus.getError());
-      else if (MaybeStatus->isDirectory())
+      if (!MaybeStatus) {
+        if (!shouldCacheStatFailures(Filename))
+          // HACK: We need to always restat non source files if the stat fails.
+          //   This is because Clang first looks up the module cache and module
+          //   files before building them, and then looks for them again. If we
+          //   cache the stat failure, it won't see them the second time.
+          return MaybeStatus.getError();
+        else
+          CacheEntry = CachedFileSystemEntry(MaybeStatus.getError());
+      } else if (MaybeStatus->isDirectory())
         CacheEntry = CachedFileSystemEntry::createDirectoryEntry(
             std::move(*MaybeStatus));
       else
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
index e5cebe381000..93bb0cde439d 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
@@ -12,8 +12,8 @@ using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
-DependencyScanningService::DependencyScanningService(ScanningMode Mode,
-                                                     bool ReuseFileManager,
-                                                     bool SkipExcludedPPRanges)
-    : Mode(Mode), ReuseFileManager(ReuseFileManager),
+DependencyScanningService::DependencyScanningService(
+    ScanningMode Mode, ScanningOutputFormat Format, bool ReuseFileManager,
+    bool SkipExcludedPPRanges)
+    : Mode(Mode), Format(Format), ReuseFileManager(ReuseFileManager),
       SkipExcludedPPRanges(SkipExcludedPPRanges) {}
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
index 82b3ae806c65..f643c538f8f9 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
@@ -1,4 +1,4 @@
-//===- DependencyScanningTool.cpp - clang-scan-deps service ------------===//
+//===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -8,18 +8,29 @@
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Frontend/Utils.h"
+#include "llvm/Support/JSON.h"
+
+static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
+  std::vector<llvm::StringRef> Strings;
+  for (auto &&I : Set)
+    Strings.push_back(I.getKey());
+  std::sort(Strings.begin(), Strings.end());
+  return llvm::json::Array(Strings);
+}
 
 namespace clang{
 namespace tooling{
 namespace dependencies{
 
-DependencyScanningTool::DependencyScanningTool(DependencyScanningService &Service,
-const tooling::CompilationDatabase &Compilations) : Worker(Service), Compilations(Compilations) {}
+DependencyScanningTool::DependencyScanningTool(
+    DependencyScanningService &Service)
+    : Format(Service.getFormat()), Worker(Service) {
+}
 
-llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(const std::string &Input,
-                                              StringRef CWD) {
+llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
+    const tooling::CompilationDatabase &Compilations, StringRef CWD) {
   /// Prints out all of the gathered dependencies into a string.
-  class DependencyPrinterConsumer : public DependencyConsumer {
+  class MakeDependencyPrinterConsumer : public DependencyConsumer {
   public:
     void handleFileDependency(const DependencyOutputOptions &Opts,
                               StringRef File) override {
@@ -28,6 +39,14 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(const std:
       Dependencies.push_back(File);
     }
 
+    void handleModuleDependency(ModuleDeps MD) override {
+      // These are ignored for the make format as it can't support the full
+      // set of deps, and handleFileDependency handles enough for implicitly
+      // built modules to work.
+    }
+
+    void handleContextHash(std::string Hash) override {}
+
     void printDependencies(std::string &S) {
       if (!Opts)
         return;
@@ -56,14 +75,98 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(const std:
     std::vector<std::string> Dependencies;
   };
 
-  DependencyPrinterConsumer Consumer;
-  auto Result =
-      Worker.computeDependencies(Input, CWD, Compilations, Consumer);
-  if (Result)
-    return std::move(Result);
-  std::string Output;
-  Consumer.printDependencies(Output);
-  return Output;
+  class FullDependencyPrinterConsumer : public DependencyConsumer {
+  public:
+    void handleFileDependency(const DependencyOutputOptions &Opts,
+                              StringRef File) override {
+      Dependencies.push_back(File);
+    }
+
+    void handleModuleDependency(ModuleDeps MD) override {
+      ClangModuleDeps[MD.ContextHash + MD.ModuleName] = std::move(MD);
+    }
+
+    void handleContextHash(std::string Hash) override {
+      ContextHash = std::move(Hash);
+    }
+
+    void printDependencies(std::string &S, StringRef MainFile) {
+      // Sort the modules by name to get a deterministic order.
+      std::vector<StringRef> Modules;
+      for (auto &&Dep : ClangModuleDeps)
+        Modules.push_back(Dep.first);
+      std::sort(Modules.begin(), Modules.end());
+
+      llvm::raw_string_ostream OS(S);
+
+      using namespace llvm::json;
+
+      Array Imports;
+      for (auto &&ModName : Modules) {
+        auto &MD = ClangModuleDeps[ModName];
+        if (MD.ImportedByMainFile)
+          Imports.push_back(MD.ModuleName);
+      }
+
+      Array Mods;
+      for (auto &&ModName : Modules) {
+        auto &MD = ClangModuleDeps[ModName];
+        Object Mod{
+            {"name", MD.ModuleName},
+            {"file-deps", toJSONSorted(MD.FileDeps)},
+            {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
+            {"clang-modulemap-file", MD.ClangModuleMapFile},
+        };
+        Mods.push_back(std::move(Mod));
+      }
+
+      Object O{
+          {"input-file", MainFile},
+          {"clang-context-hash", ContextHash},
+          {"file-deps", Dependencies},
+          {"clang-module-deps", std::move(Imports)},
+          {"clang-modules", std::move(Mods)},
+      };
+
+      S = llvm::formatv("{0:2},\n", Value(std::move(O))).str();
+      return;
+    }
+
+  private:
+    std::vector<std::string> Dependencies;
+    std::unordered_map<std::string, ModuleDeps> ClangModuleDeps;
+    std::string ContextHash;
+  };
+
+  
+  // We expect a single command here because if a source file occurs multiple
+  // times in the original CDB, then `computeDependencies` would run the
+  // `DependencyScanningAction` once for every time the input occured in the
+  // CDB. Instead we split up the CDB into single command chunks to avoid this
+  // behavior.
+  assert(Compilations.getAllCompileCommands().size() == 1 &&
+         "Expected a compilation database with a single command!");
+  std::string Input = Compilations.getAllCompileCommands().front().Filename;
+  
+  if (Format == ScanningOutputFormat::Make) {
+    MakeDependencyPrinterConsumer Consumer;
+    auto Result =
+        Worker.computeDependencies(Input, CWD, Compilations, Consumer);
+    if (Result)
+      return std::move(Result);
+    std::string Output;
+    Consumer.printDependencies(Output);
+    return Output;
+  } else {
+    FullDependencyPrinterConsumer Consumer;
+    auto Result =
+        Worker.computeDependencies(Input, CWD, Compilations, Consumer);
+    if (Result)
+      return std::move(Result);
+    std::string Output;
+    Consumer.printDependencies(Output, Input);
+    return Output;
+  }
 }
 
 } // end namespace dependencies
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index f382c202f8c2..edf2cf8bd70f 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -14,6 +14,7 @@
 #include "clang/Frontend/Utils.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
 #include "clang/Tooling/Tooling.h"
 
 using namespace clang;
@@ -72,9 +73,11 @@ public:
   DependencyScanningAction(
       StringRef WorkingDirectory, DependencyConsumer &Consumer,
       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
-      ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings)
+      ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
+      ScanningOutputFormat Format)
       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
-        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings) {}
+        DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings),
+        Format(Format) {}
 
   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
                      FileManager *FileMgr,
@@ -131,9 +134,20 @@ public:
     // We need at least one -MT equivalent for the generator to work.
     if (Opts->Targets.empty())
       Opts->Targets = {"clang-scan-deps dependency"};
-    Compiler.addDependencyCollector(
-        std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
-                                                      Consumer));
+
+    switch (Format) {
+    case ScanningOutputFormat::Make:
+      Compiler.addDependencyCollector(
+          std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
+                                                        Consumer));
+      break;
+    case ScanningOutputFormat::Full:
+      Compiler.addDependencyCollector(
+          std::make_shared<ModuleDepCollector>(Compiler, Consumer));
+      break;
+    }
+
+    Consumer.handleContextHash(Compiler.getInvocation().getModuleHash());
 
     auto Action = std::make_unique<PreprocessOnlyAction>();
     const bool Result = Compiler.ExecuteAction(*Action);
@@ -147,12 +161,14 @@ private:
   DependencyConsumer &Consumer;
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
+  ScanningOutputFormat Format;
 };
 
 } // end anonymous namespace
 
 DependencyScanningWorker::DependencyScanningWorker(
-    DependencyScanningService &Service) {
+    DependencyScanningService &Service)
+    : Format(Service.getFormat()) {
   DiagOpts = new DiagnosticOptions();
   PCHContainerOps = std::make_shared<PCHContainerOperations>();
   RealFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem());
@@ -195,7 +211,7 @@ llvm::Error DependencyScanningWorker::computeDependencies(
     Tool.setPrintErrorMessage(false);
     Tool.setDiagnosticConsumer(&DC);
     DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
-                                    PPSkipMappings.get());
+                                    PPSkipMappings.get(), Format);
     return !Tool.run(&Action);
   });
 }
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
new file mode 100644
index 000000000000..422940047f2d
--- /dev/null
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -0,0 +1,135 @@
+//===- ModuleDepCollector.cpp - Callbacks to collect deps -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
+
+using namespace clang;
+using namespace tooling;
+using namespace dependencies;
+
+void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
+                                       FileChangeReason Reason,
+                                       SrcMgr::CharacteristicKind FileType,
+                                       FileID PrevFID) {
+  if (Reason != PPCallbacks::EnterFile)
+    return;
+
+  SourceManager &SM = Instance.getSourceManager();
+
+  // Dependency generation really does want to go all the way to the
+  // file entry for a source location to find out what is depended on.
+  // We do not want #line markers to affect dependency generation!
+  Optional<FileEntryRef> File =
+      SM.getFileEntryRefForID(SM.getFileID(SM.getExpansionLoc(Loc)));
+  if (!File)
+    return;
+
+  StringRef FileName =
+      llvm::sys::path::remove_leading_dotslash(File->getName());
+
+  MDC.MainDeps.push_back(FileName);
+}
+
+void ModuleDepCollectorPP::InclusionDirective(
+    SourceLocation HashLoc, const Token &IncludeTok, StringRef FileName,
+    bool IsAngled, CharSourceRange FilenameRange, const FileEntry *File,
+    StringRef SearchPath, StringRef RelativePath, const Module *Imported,
+    SrcMgr::CharacteristicKind FileType) {
+  if (!File && !Imported) {
+    // This is a non-modular include that HeaderSearch failed to find. Add it
+    // here as `FileChanged` will never see it.
+    MDC.MainDeps.push_back(FileName);
+  }
+
+  if (!Imported)
+    return;
+
+  MDC.Deps[MDC.ContextHash + Imported->getTopLevelModule()->getFullModuleName()]
+      .ImportedByMainFile = true;
+  DirectDeps.insert(Imported->getTopLevelModule());
+}
+
+void ModuleDepCollectorPP::EndOfMainFile() {
+  FileID MainFileID = Instance.getSourceManager().getMainFileID();
+  MDC.MainFile =
+      Instance.getSourceManager().getFileEntryForID(MainFileID)->getName();
+
+  for (const Module *M : DirectDeps) {
+    handleTopLevelModule(M);
+  }
+
+  for (auto &&I : MDC.Deps)
+    MDC.Consumer.handleModuleDependency(I.second);
+
+  DependencyOutputOptions Opts;
+  for (auto &&I : MDC.MainDeps)
+    MDC.Consumer.handleFileDependency(Opts, I);
+}
+
+void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
+  assert(M == M->getTopLevelModule() && "Expected top level module!");
+
+  auto ModI = MDC.Deps.insert(
+      std::make_pair(MDC.ContextHash + M->getFullModuleName(), ModuleDeps{}));
+
+  if (!ModI.first->second.ModuleName.empty())
+    return;
+
+  ModuleDeps &MD = ModI.first->second;
+
+  const FileEntry *ModuleMap = Instance.getPreprocessor()
+                                   .getHeaderSearchInfo()
+                                   .getModuleMap()
+                                   .getContainingModuleMapFile(M);
+
+  MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : "";
+  MD.ModuleName = M->getFullModuleName();
+  MD.ModulePCMPath = M->getASTFile()->getName();
+  MD.ContextHash = MDC.ContextHash;
+  serialization::ModuleFile *MF =
+      MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile());
+  MDC.Instance.getASTReader()->visitInputFiles(
+      *MF, true, true, [&](const serialization::InputFile &IF, bool isSystem) {
+        MD.FileDeps.insert(IF.getFile()->getName());
+      });
+
+  addAllSubmoduleDeps(M, MD);
+}
+
+void ModuleDepCollectorPP::addAllSubmoduleDeps(const Module *M,
+                                               ModuleDeps &MD) {
+  addModuleDep(M, MD);
+
+  for (const Module *SubM : M->submodules())
+    addAllSubmoduleDeps(SubM, MD);
+}
+
+void ModuleDepCollectorPP::addModuleDep(const Module *M, ModuleDeps &MD) {
+  for (const Module *Import : M->Imports) {
+    if (Import->getTopLevelModule() != M->getTopLevelModule()) {
+      MD.ClangModuleDeps.insert(Import->getTopLevelModuleName());
+      handleTopLevelModule(Import->getTopLevelModule());
+    }
+  }
+}
+
+ModuleDepCollector::ModuleDepCollector(CompilerInstance &I,
+                                       DependencyConsumer &C)
+    : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()) {
+}
+
+void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
+  PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(Instance, *this));
+}
+
+void ModuleDepCollector::attachToASTReader(ASTReader &R) {}