[Frontend][Parse][Sema] Add support for literate input files.

include/swift/Basic/FileTypes.def

@@ -39,6 +39,9 @@
 TYPE("swift",               Swift,                     "swift",           "")
 TYPE("sil",                 SIL,                       "sil",             "")
 TYPE("sib",                 SIB,                       "sib",             "")
+TYPE("markdown",            Markdown,                  "md",              "")
+TYPE("restructured-text",   reStructuredText,          "rst",             "")
+TYPE("latex",               LaTeX,                     "tex",             "")
 
 // Output types
 TYPE("ast-dump",            ASTDump,                   "ast",             "")

include/swift/Basic/FileTypes.h

@@ -45,6 +45,10 @@ ID lookupTypeForName(StringRef Name);
 /// Returns true if the type represents textual data.
 bool isTextual(ID Id);
 
+/// Returns true if the type is a Swift source file.  This includes literate
+/// Swift, which might be in a Markdown, reStructuredText or LaTeX wrapper.
+bool isSwiftSourceCode(ID Id);
+
 /// Returns true if the type is produced in the compiler after the LLVM
 /// passes.
 ///

include/swift/Driver/Action.h

@@ -203,14 +203,14 @@ class CompileJobAction : public IncrementalJobAction {
     return A->getKind() == Action::Kind::CompileJob;
   }
 
-  /// Return a _single_ TY_Swift InputAction, if one exists;
+  /// Return a _single_ TY_Swift/Markdown/etc. InputAction, if one exists;
   /// if 0 or >1 such inputs exist, return nullptr.
   const InputAction *findSingleSwiftInput() const {
     auto Inputs = getInputs();
     const InputAction *IA = nullptr;
     for (auto const *I : Inputs) {
       if (auto const *S = dyn_cast<InputAction>(I)) {
-        if (S->getType() == file_types::TY_Swift) {
+        if (file_types::isSwiftSourceCode(S->getType())) {
           if (IA == nullptr) {
             IA = S;
           } else {

include/swift/Driver/ToolChain.h

@@ -272,8 +272,8 @@ class ToolChain {
   /// batching together into a BatchJob, via a call to \c
   /// constructBatchJob. This is true when the \c Job is a built from a \c
   /// CompileJobAction in a \c Compilation \p C running in \c
-  /// OutputInfo::Mode::StandardCompile output mode, with a single \c TY_Swift
-  /// \c InputAction.
+  /// OutputInfo::Mode::StandardCompile output mode, with a single \c TY_Swift,
+  /// \c TY_Markdown, \c TY_reStructuredText or \c TY_LaTeX \c InputAction.
   bool jobIsBatchable(const Compilation &C, const Job *A) const;
 
   /// Equivalence relation that holds iff the two input Jobs \p A and \p B are

include/swift/Parse/Lexer.h

@@ -57,6 +57,9 @@ enum class HashbangMode : bool {
 
 enum class LexerMode {
   Swift,
+  Markdown,
+  reStructuredText,
+  LaTeX,
   SwiftInterface,
   SIL
 };
@@ -102,8 +105,12 @@ class Lexer {
   /// Pointer to the next not consumed character.
   const char *CurPtr;
 
+  /// Number of spaces of indentation on the current line.
+  unsigned Indentation;
+  bool ScanningIndent;
+
   Token NextToken;
-  
+
   /// The kind of source we're lexing. This either enables special behavior for
   /// module interfaces, or enables things like the 'sil' keyword if lexing
   /// a .sil file.
@@ -120,6 +127,26 @@ class Lexer {
   /// in a SIL file.  This enables some context-sensitive lexing.
   bool InSILBody = false;
 
+  /// InCodeBlock - When in literate mode, this is false when we're outside a
+  /// code block.
+  bool InCodeBlock = true;
+
+  /// Markdown code blocks are delimited by code fences, which is a sequence of
+  /// at least three consecutive backtick characters or tildes, indented by no
+  /// more than three spaces.  Backticks and tildes cannot be mixed, and the
+  /// closing fence must have at least the same number as the opening fence.
+  /// Keep track of Markdown fence information here.
+  char MarkdownFence;
+  unsigned MarkdownFenceLength;
+
+  /// reStructuredText code blocks are handled using indentation; they start
+  /// after *either* a "::" at the end of a paragraph, *or* a ".. code-block::"
+  /// or ".. sourcecode::" block, and continue until you return to the previous
+  /// indent level.
+  ///
+  /// This means we need to track the indent level of preceding paragraphs.
+  unsigned RSTIndentLevel;
+
   /// The current leading trivia for the next token.
   ///
   /// The StringRef points into the source buffer that is currently being lexed.
@@ -521,6 +548,15 @@ class Lexer {
     return BufferStart + SourceMgr.getLocOffsetInBuffer(Loc, BufferID);
   }
 
+  void rewindToStartOfLine();
+  void skipRSTCodeBlock();
+  StringRef scanWord(char ExtraTerm=0);
+  void advanceToMarkdownCodeBlock();
+  void advanceToRSTCodeBlock();
+  void advanceToLaTeXCodeBlock();
+  void advanceToCodeBlock();
+  bool atEndOfCodeBlock();
+
   void lexImpl();
   InFlightDiagnostic diagnose(const char *Loc, Diagnostic Diag);
 

lib/Basic/FileTypes.cpp

@@ -67,6 +67,9 @@ ID file_types::lookupTypeForName(StringRef Name) {
 bool file_types::isTextual(ID Id) {
   switch (Id) {
   case file_types::TY_Swift:
+  case file_types::TY_Markdown:
+  case file_types::TY_reStructuredText:
+  case file_types::TY_LaTeX:
   case file_types::TY_SIL:
   case file_types::TY_Dependencies:
   case file_types::TY_Assembly:
@@ -115,6 +118,18 @@ bool file_types::isTextual(ID Id) {
   llvm_unreachable("All switch cases are covered");
 }
 
+bool file_types::isSwiftSourceCode(ID Id) {
+  switch (Id) {
+  case file_types::TY_Swift:
+  case file_types::TY_Markdown:
+  case file_types::TY_reStructuredText:
+  case file_types::TY_LaTeX:
+    return true;
+  default:
+    return false;
+  }
+}
+
 bool file_types::isAfterLLVM(ID Id) {
   switch (Id) {
   case file_types::TY_Assembly:
@@ -123,6 +138,9 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_Object:
     return true;
   case file_types::TY_Swift:
+  case file_types::TY_Markdown:
+  case file_types::TY_reStructuredText:
+  case file_types::TY_LaTeX:
   case file_types::TY_PCH:
   case file_types::TY_ImportedModules:
   case file_types::TY_TBD:
@@ -169,6 +187,9 @@ bool file_types::isAfterLLVM(ID Id) {
 bool file_types::isPartOfSwiftCompilation(ID Id) {
   switch (Id) {
   case file_types::TY_Swift:
+  case file_types::TY_Markdown:
+  case file_types::TY_reStructuredText:
+  case file_types::TY_LaTeX:
   case file_types::TY_SIL:
   case file_types::TY_RawSIL:
   case file_types::TY_SIB:

lib/Driver/Compilation.cpp

@@ -1905,7 +1905,7 @@ void Compilation::disableIncrementalBuild(Twine why) {
 unsigned Compilation::countSwiftInputs() const {
   unsigned inputCount = 0;
   for (const auto &p : InputFilesWithTypes)
-    if (p.first == file_types::TY_Swift)
+    if (file_types::isSwiftSourceCode(p.first))
       ++inputCount;
   return inputCount;
 }

lib/Driver/Driver.cpp

@@ -1325,7 +1325,7 @@ void Driver::buildInputs(const ToolChain &TC,
       if (HasVFS || checkInputExistence(*this, Args, Diags, Value))
         Inputs.push_back(std::make_pair(Ty, A));
 
-      if (Ty == file_types::TY_Swift) {
+      if (file_types::isSwiftSourceCode(Ty)) {
         StringRef Basename = llvm::sys::path::filename(Value);
         if (!SourceFileNames.insert({Basename, Value}).second) {
           Diags.diagnose(SourceLoc(), diag::error_two_files_same_name,
@@ -1975,6 +1975,9 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       Action *Current = C.createAction<InputAction>(*InputArg, InputType);
       switch (InputType) {
       case file_types::TY_Swift:
+      case file_types::TY_Markdown:
+      case file_types::TY_reStructuredText:
+      case file_types::TY_LaTeX:
       case file_types::TY_SIL:
       case file_types::TY_SIB: {
         // Source inputs always need to be compiled.

lib/Driver/ToolChains.cpp

@@ -646,6 +646,9 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
       llvm_unreachable("We were told to perform a standard compile, "
                        "but no mode option was passed to the driver.");
   case file_types::TY_Swift:
+  case file_types::TY_Markdown:
+  case file_types::TY_reStructuredText:
+  case file_types::TY_LaTeX:
   case file_types::TY_dSYM:
   case file_types::TY_AutolinkFile:
   case file_types::TY_Dependencies:
@@ -905,6 +908,9 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case file_types::TY_JSONFeatures:
       llvm_unreachable("Cannot be output from backend job");
     case file_types::TY_Swift:
+    case file_types::TY_Markdown:
+    case file_types::TY_reStructuredText:
+    case file_types::TY_LaTeX:
     case file_types::TY_dSYM:
     case file_types::TY_AutolinkFile:
     case file_types::TY_Dependencies:

lib/Frontend/Frontend.cpp

@@ -657,10 +657,34 @@ CompilerInstance::getRecordedBufferID(const InputFile &input,
   auto buffers = getInputBuffersIfPresent(input);
 
   // Recover by dummy buffer if requested.
-  if (!buffers.hasValue() && shouldRecover &&
-      input.getType() == file_types::TY_Swift) {
-    buffers = ModuleBuffers(llvm::MemoryBuffer::getMemBuffer(
-        "// missing file\n", input.getFileName()));
+  if (!buffers.hasValue() && shouldRecover) {
+    switch (input.getType()) {
+    case file_types::TY_Swift:
+      buffers = ModuleBuffers(llvm::MemoryBuffer::getMemBuffer(
+                                "// missing file\n", input.getFileName()));
+      break;
+    case file_types::TY_Markdown:
+    case file_types::TY_reStructuredText:
+      buffers = ModuleBuffers(llvm::MemoryBuffer::getMemBuffer(
+                                "Missing file\n"
+                                "============\n"
+                                "\n"
+                                "Missing file\n",
+                                input.getFileName()));
+      break;
+    case file_types::TY_LaTeX:
+      buffers = ModuleBuffers(llvm::MemoryBuffer::getMemBuffer(
+                                "\\documentclass{article}\n"
+                                "\\title{Missing file}\n"
+                                "\\begin{document}\n"
+                                "  \\maketitle\n"
+                                "  Missing file\n"
+                                "\\end{document}\n",
+                                input.getFileName()));
+      break;
+    default:
+      break;
+    }
   }
 
   if (!buffers.hasValue()) {
@@ -891,11 +915,22 @@ CompilerInstance::computeMainSourceFileForModule(ModuleDecl *mod) const {
     return nullptr;
   }
 
-  // Try to pull out a file called 'main.swift'.
+  // Try to pull out a file called 'main.(swift|md|rst|tex)'.
   auto MainInputIter =
       std::find_if(Inputs.begin(), Inputs.end(), [](const InputFile &input) {
-        return input.getType() == file_types::TY_Swift &&
-               llvm::sys::path::filename(input.getFileName()) == "main.swift";
+        auto filename = llvm::sys::path::filename(input.getFileName());
+        switch (input.getType()) {
+        case file_types::TY_Swift:
+          return filename == "main.swift";
+        case file_types::TY_Markdown:
+          return filename == "main.md";
+        case file_types::TY_reStructuredText:
+          return filename == "main.rst";
+        case file_types::TY_LaTeX:
+          return filename == "main.tex";
+        default:
+          return false;
+        }
       });
 
   Optional<unsigned> MainBufferID = None;

lib/FrontendTool/FrontendTool.cpp

@@ -1250,7 +1250,7 @@ static bool performCompile(CompilerInstance &Instance,
       return llvm::all_of(
           opts.InputsAndOutputs.getAllInputs(), [](const InputFile &IF) {
             const auto kind = IF.getType();
-            return kind == file_types::TY_Swift ||
+            return file_types::isSwiftSourceCode(kind) ||
                    kind == file_types::TY_SwiftModuleInterfaceFile;
           });
     }