[Tooling/DependencyScanning] Rename refactorings towards transitioning dependency scanning to use pre-lexed preprocessor directive tokens

This is first of a series of patches for making the special lexing for dependency scanning a first-class feature of the `Preprocessor` and `Lexer`.
This patch only includes NFC renaming changes to make reviewing of the functionality changing parts easier.

Differential Revision: https://reviews.llvm.org/D125484

Author: akyrtzi

clang/include/clang/Basic/DiagnosticLexKinds.td

@@ -899,11 +899,11 @@ def err_pp_eof_in_assume_nonnull : Error<
 
 }
 
-let CategoryName = "Dependency Directive Source Minimization Issue" in {
+let CategoryName = "Dependency Directive Source Scanner Issue" in {
 
-def err_dep_source_minimizer_missing_sema_after_at_import : Error<
+def err_dep_source_scanner_missing_semi_after_at_import : Error<
   "could not find ';' after @import">;
-def err_dep_source_minimizer_unexpected_tokens_at_import : Error<
+def err_dep_source_scanner_unexpected_tokens_at_import : Error<
   "unexpected extra tokens at end of @import declaration">;
 
 }

clang/include/clang/Lex/DependencyDirectivesSourceMinimizer.h renamed to clang/include/clang/Lex/DependencyDirectivesScanner.h

@@ -1,4 +1,4 @@
-//===- clang/Lex/DependencyDirectivesSourceMinimizer.h -  ----------*- C++ -*-//
+//===- clang/Lex/DependencyDirectivesScanner.h ---------------------*- C++ -*-//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,15 +7,15 @@
 //===----------------------------------------------------------------------===//
 ///
 /// \file
-/// This is the interface for minimizing header and source files to the
+/// This is the interface for scanning header and source files to get the
 /// minimum necessary preprocessor directives for evaluating includes. It
 /// reduces the source down to #define, #include, #import, @import, and any
 /// conditional preprocessor logic that contains one of those.
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSOURCEMINIMIZER_H
-#define LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSOURCEMINIMIZER_H
+#ifndef LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H
+#define LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H
 
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -26,11 +26,11 @@ namespace clang {
 
 class DiagnosticsEngine;
 
-namespace minimize_source_to_dependency_directives {
+namespace dependency_directives_scan {
 
 /// Represents the kind of preprocessor directive or a module declaration that
-/// is tracked by the source minimizer in its token output.
-enum TokenKind {
+/// is tracked by the scanner in its token output.
+enum DirectiveKind : uint8_t {
   pp_none,
   pp_include,
   pp___include_macros,
@@ -58,17 +58,17 @@ enum TokenKind {
   pp_eof,
 };
 
-/// Represents a simplified token that's lexed as part of the source
-/// minimization. It's used to track the location of various preprocessor
-/// directives that could potentially have an effect on the depedencies.
-struct Token {
+/// Represents a directive that's lexed as part of the dependency directives
+/// scanning. It's used to track various preprocessor directives that could
+/// potentially have an effect on the depedencies.
+struct Directive {
   /// The kind of token.
-  TokenKind K = pp_none;
+  DirectiveKind Kind = pp_none;
 
   /// Offset into the output byte stream of where the directive begins.
   int Offset = -1;
 
-  Token(TokenKind K, int Offset) : K(K), Offset(Offset) {}
+  Directive(DirectiveKind K, int Offset) : Kind(K), Offset(Offset) {}
 };
 
 /// Simplified token range to track the range of a potentially skippable PP
@@ -86,10 +86,10 @@ struct SkippedRange {
 /// when skipping a directive like #if, #ifdef or #elsif.
 ///
 /// \returns false on success, true on error.
-bool computeSkippedRanges(ArrayRef<Token> Input,
+bool computeSkippedRanges(ArrayRef<Directive> Input,
                           llvm::SmallVectorImpl<SkippedRange> &Range);
 
-} // end namespace minimize_source_to_dependency_directives
+} // end namespace dependency_directives_scan
 
 /// Minimize the input down to the preprocessor directives that might have
 /// an effect on the dependencies for a compilation unit.
@@ -103,13 +103,12 @@ bool computeSkippedRanges(ArrayRef<Token> Input,
 /// \returns false on success, true on error. If the diagnostic engine is not
 /// null, an appropriate error is reported using the given input location
 /// with the offset that corresponds to the minimizer's current buffer offset.
-bool minimizeSourceToDependencyDirectives(
+bool scanSourceForDependencyDirectives(
     llvm::StringRef Input, llvm::SmallVectorImpl<char> &Output,
-    llvm::SmallVectorImpl<minimize_source_to_dependency_directives::Token>
-        &Tokens,
+    llvm::SmallVectorImpl<dependency_directives_scan::Directive> &Directives,
     DiagnosticsEngine *Diags = nullptr,
     SourceLocation InputSourceLoc = SourceLocation());
 
 } // end namespace clang
 
-#endif // LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSOURCEMINIMIZER_H
+#endif // LLVM_CLANG_LEX_DEPENDENCYDIRECTIVESSCANNER_H

clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h

@@ -28,14 +28,14 @@ struct CachedFileContents {
   CachedFileContents(std::unique_ptr<llvm::MemoryBuffer> Original)
       : Original(std::move(Original)), MinimizedAccess(nullptr) {}
 
-  /// Owning storage for the minimized contents.
+  /// Owning storage for the original contents.
   std::unique_ptr<llvm::MemoryBuffer> Original;
 
-  /// The mutex that must be locked before mutating minimized contents.
+  /// The mutex that must be locked before mutating directive tokens.
   std::mutex ValueLock;
   /// Owning storage for the minimized contents.
   std::unique_ptr<llvm::MemoryBuffer> MinimizedStorage;
-  /// Accessor to the minimized contents that's atomic to avoid data races.
+  /// Accessor to the directive tokens that's atomic to avoid data races.
   std::atomic<llvm::MemoryBuffer *> MinimizedAccess;
   /// Skipped range mapping of the minimized contents.
   /// This is initialized iff `MinimizedAccess != nullptr`.
@@ -46,8 +46,8 @@ struct CachedFileContents {
 /// the dependency scanning filesystem.
 ///
 /// It represents one of the following:
-/// - opened file with original contents and a stat value,
-/// - opened file with original contents, minimized contents and a stat value,
+/// - opened file with contents and a stat value,
+/// - opened file with contents, directive tokens and a stat value,
 /// - directory entry with its stat value,
 /// - filesystem error.
 ///
@@ -84,8 +84,9 @@ class CachedFileSystemEntry {
     return Contents->Original->getBuffer();
   }
 
-  /// \returns Minimized contents of the file.
-  StringRef getMinimizedContents() const {
+  /// \returns The scanned preprocessor directive tokens of the file that are
+  /// used to speed up preprocessing, if available.
+  StringRef getDirectiveTokens() const {
     assert(!isError() && "error");
     assert(!MaybeStat->isDirectory() && "not a file");
     assert(Contents && "contents not initialized");
@@ -119,8 +120,8 @@ class CachedFileSystemEntry {
     return Contents->PPSkippedRangeMapping;
   }
 
-  /// \returns The data structure holding both original and minimized contents.
-  CachedFileContents *getContents() const {
+  /// \returns The data structure holding both contents and directive tokens.
+  CachedFileContents *getCachedContents() const {
     assert(!isError() && "error");
     assert(!isDirectory() && "not a file");
     return Contents;
@@ -145,7 +146,7 @@ class CachedFileSystemEntry {
 };
 
 /// This class is a shared cache, that caches the 'stat' and 'open' calls to the
-/// underlying real file system. It distinguishes between minimized and original
+/// underlying real file system, and the scanned preprocessor directives of
 /// files.
 ///
 /// It is sharded based on the hash of the key to reduce the lock contention for
@@ -210,8 +211,7 @@ class DependencyScanningFilesystemSharedCache {
 };
 
 /// This class is a local cache, that caches the 'stat' and 'open' calls to the
-/// underlying real file system. It distinguishes between minimized and original
-/// files.
+/// underlying real file system.
 class DependencyScanningFilesystemLocalCache {
   llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
 
@@ -234,9 +234,8 @@ class DependencyScanningFilesystemLocalCache {
 };
 
 /// Reference to a CachedFileSystemEntry.
-/// If the underlying entry is an opened file, this wrapper returns the correct
-/// contents (original or minimized) and ensures consistency with file size
-/// reported by status.
+/// If the underlying entry is an opened file, this wrapper returns the file
+/// contents and the scanned preprocessor directives.
 class EntryRef {
   /// For entry that is an opened file, this bit signifies whether its contents
   /// are minimized.
@@ -270,8 +269,7 @@ class EntryRef {
   }
 
   StringRef getContents() const {
-    return Minimized ? Entry.getMinimizedContents()
-                     : Entry.getOriginalContents();
+    return Minimized ? Entry.getDirectiveTokens() : Entry.getOriginalContents();
   }
 
   const PreprocessorSkippedRangeMapping *getPPSkippedRangeMapping() const {
@@ -301,33 +299,33 @@ class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
   openFileForRead(const Twine &Path) override;
 
-  /// Disable minimization of the given file.
-  void disableMinimization(StringRef Filename);
-  /// Enable minimization of all files.
-  void enableMinimizationOfAllFiles() { NotToBeMinimized.clear(); }
+  /// Disable directives scanning of the given file.
+  void disableDirectivesScanning(StringRef Filename);
+  /// Enable directives scanning of all files.
+  void enableDirectivesScanningOfAllFiles() { NotToBeScanned.clear(); }
 
 private:
-  /// Check whether the file should be minimized.
-  bool shouldMinimize(StringRef Filename, llvm::sys::fs::UniqueID UID);
+  /// Check whether the file should be scanned for preprocessor directives.
+  bool shouldScanForDirectives(StringRef Filename, llvm::sys::fs::UniqueID UID);
 
   /// Returns entry for the given filename.
   ///
   /// Attempts to use the local and shared caches first, then falls back to
   /// using the underlying filesystem.
   llvm::ErrorOr<EntryRef>
   getOrCreateFileSystemEntry(StringRef Filename,
-                             bool DisableMinimization = false);
+                             bool DisableDirectivesScanning = false);
 
   /// For a filename that's not yet associated with any entry in the caches,
   /// uses the underlying filesystem to either look up the entry based in the
   /// shared cache indexed by unique ID, or creates new entry from scratch.
   llvm::ErrorOr<const CachedFileSystemEntry &>
   computeAndStoreResult(StringRef Filename);
 
-  /// Minimizes the given entry if necessary and returns a wrapper object with
-  /// reference semantics.
-  EntryRef minimizeIfNecessary(const CachedFileSystemEntry &Entry,
-                               StringRef Filename, bool Disable);
+  /// Scan for preprocessor directives for the given entry if necessary and
+  /// returns a wrapper object with reference semantics.
+  EntryRef scanForDirectivesIfNecessary(const CachedFileSystemEntry &Entry,
+                                        StringRef Filename, bool Disable);
 
   /// Represents a filesystem entry that has been stat-ed (and potentially read)
   /// and that's about to be inserted into the cache as `CachedFileSystemEntry`.
@@ -402,8 +400,8 @@ class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
   /// excluded conditional directive skip mappings that are used by the
   /// currently active preprocessor.
   ExcludedPreprocessorDirectiveSkipMapping &PPSkipMappings;
-  /// The set of files that should not be minimized.
-  llvm::DenseSet<llvm::sys::fs::UniqueID> NotToBeMinimized;
+  /// The set of files that should not be scanned for PP directives.
+  llvm::DenseSet<llvm::sys::fs::UniqueID> NotToBeScanned;
 };
 
 } // end namespace dependencies

clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h

@@ -19,15 +19,13 @@ namespace dependencies {
 /// dependencies.
 enum class ScanningMode {
   /// This mode is used to compute the dependencies by running the preprocessor
-  /// over
-  /// the unmodified source files.
+  /// over the source files.
   CanonicalPreprocessing,
 
   /// This mode is used to compute the dependencies by running the preprocessor
-  /// over
-  /// the source files that have been minimized to contents that might affect
-  /// the dependencies.
-  MinimizedSourcePreprocessing
+  /// with special kind of lexing after scanning header and source files to get
+  /// the minimum necessary preprocessor directives for evaluating includes.
+  DependencyDirectivesScan,
 };
 
 /// The format that is output by the dependency scanner.

clang/lib/Frontend/FrontendActions.cpp

@@ -18,7 +18,7 @@
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Frontend/Utils.h"
-#include "clang/Lex/DependencyDirectivesSourceMinimizer.h"
+#include "clang/Lex/DependencyDirectivesScanner.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
@@ -1158,9 +1158,9 @@ void PrintDependencyDirectivesSourceMinimizerAction::ExecuteAction() {
   llvm::MemoryBufferRef FromFile = SM.getBufferOrFake(SM.getMainFileID());
 
   llvm::SmallString<1024> Output;
-  llvm::SmallVector<minimize_source_to_dependency_directives::Token, 32> Toks;
-  if (minimizeSourceToDependencyDirectives(
-          FromFile.getBuffer(), Output, Toks, &CI.getDiagnostics(),
+  llvm::SmallVector<dependency_directives_scan::Directive, 32> Directives;
+  if (scanSourceForDependencyDirectives(
+          FromFile.getBuffer(), Output, Directives, &CI.getDiagnostics(),
           SM.getLocForStartOfFile(SM.getMainFileID()))) {
     assert(CI.getDiagnostics().hasErrorOccurred() &&
            "no errors reported for failure");

clang/lib/Lex/CMakeLists.txt

@@ -3,7 +3,7 @@
 set(LLVM_LINK_COMPONENTS support)
 
 add_clang_library(clangLex
-  DependencyDirectivesSourceMinimizer.cpp
+  DependencyDirectivesScanner.cpp
   HeaderMap.cpp
   HeaderSearch.cpp
   InitHeaderSearch.cpp