Merge branch 'main' into github-issue

Author: Kyle-Ye

.github/PULL_REQUEST_TEMPLATE/CHERRY_PICK.md

@@ -0,0 +1,6 @@
+- **Explanation**: _A description of the issue being fixed or enhancement being made. This can be brief, but it should be clear._
+- **Scope**: _An assessment of the impact/importance of the change. For example, is the change a source-breaking language change, etc._
+- **GitHub Issue**: _The issue number if the change fixes/implements an issue/enhancement on [GitHub Issues](https://github.com/apple/swift-docc/issues)._
+- **Risk**: _What is the (specific) risk to the release for taking this change?_
+- **Testing**: _What specific testing has been done or needs to be done to further validate any impact of this change?_
+- **Reviewer**: _One or more code owners for the impacted components should review the change. Technical review can be delegated by a code owner or otherwise requested as deemed appropriate or useful._

.github/pull_request_template.md

@@ -1,3 +1,8 @@
+<!--
+If you're opening a PR to cherry-pick a change for a release branch, use this template instead:
+https://github.com/apple/swift-docc/blob/main/.github/PULL_REQUEST_TEMPLATE/CHERRY_PICK.md
+-->
+
 Bug/issue #, if applicable: 
 
 ## Summary

.gitignore

@@ -22,3 +22,4 @@ DerivedData
 docc-dist-build
 .docc-build
 .swiftpm
+.vscode

Sources/SwiftDocC/Indexing/Navigator/NavigatorIndex.swift

@@ -910,25 +910,32 @@ extension NavigatorIndex {
             // Assign the children to the parents, starting with multi curated nodes
             var nodesMultiCurated = multiCurated.map { ($0, $1) }
 
-            for index in 0..<nodesMultiCurated.count {
-                let (nodeID, parent) = nodesMultiCurated[index]
-                let placeholders = identifierToChildren[nodeID]!
-                for reference in placeholders {
-                    if let child = identifierToNode[reference] {
-                        parent.add(child: child)
-                        pendingUncuratedReferences.remove(reference)
-                        if !multiCurated.keys.contains(reference) && reference.fragment == nil {
-                            // As the children of a multi-curated node is itself curated multiple times
-                            // we need to process it as well, ignoring items with fragments as those are sections.
-                            nodesMultiCurated.append((reference, child))
-                            multiCurated[reference] = child
+            while !nodesMultiCurated.isEmpty {
+                // The children of the multicurated nodes. These need to be tracked so we can multicurate them as well.
+                var nodesMultiCuratedChildren: [(Identifier, NavigatorTree.Node)] = []
+                
+                for index in 0..<nodesMultiCurated.count {
+                    let (nodeID, parent) = nodesMultiCurated[index]
+                    let placeholders = identifierToChildren[nodeID]!
+                    for reference in placeholders {
+                        if let child = identifierToNode[reference] {
+                            parent.add(child: child)
+                            pendingUncuratedReferences.remove(reference)
+                            if !multiCurated.keys.contains(reference) && reference.fragment == nil {
+                                // As the children of a multi-curated node is itself curated multiple times
+                                // we need to process it as well, ignoring items with fragments as those are sections.
+                                nodesMultiCuratedChildren.append((reference, child))
+                                multiCurated[reference] = child
+                            }
                         }
                     }
+                    // Once assigned, placeholders can be removed as we use copy later.
+                    identifierToChildren[nodeID]!.removeAll()
                 }
-                // Once assigned, placeholders can be removed as we use copy later.
-                identifierToChildren[nodeID]!.removeAll()
+                
+                nodesMultiCurated = nodesMultiCuratedChildren
             }
-            
+                
             for (nodeIdentifier, placeholders) in identifierToChildren {
                 for reference in placeholders {
                     let parent = identifierToNode[nodeIdentifier]!

Sources/SwiftDocC/Infrastructure/DocumentationContext.swift

@@ -312,6 +312,9 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     ///   - bundle: The bundle that was added.
     public func dataProvider(_ dataProvider: DocumentationContextDataProvider, didAddBundle bundle: DocumentationBundle) throws {
         try benchmark(wrap: Benchmark.Duration(id: "bundle-registration")) {
+            // Enable reference caching for this documentation bundle.
+            ResolvedTopicReference.enableReferenceCaching(for: bundle.identifier)
+            
             try self.register(bundle)
         }
     }
@@ -323,7 +326,11 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     ///   - bundle: The bundle that was removed.
     public func dataProvider(_ dataProvider: DocumentationContextDataProvider, didRemoveBundle bundle: DocumentationBundle) throws {
         referenceCache.sync { $0.removeAll() }
+        
+        // Purge the reference cache for this bundle and disable reference caching for
+        // this bundle moving forward.
         ResolvedTopicReference.purgePool(for: bundle.identifier)
+        
         unregister(bundle)
     }
 
@@ -1561,28 +1568,24 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     /// - ``SymbolGraphRelationshipsBuilder``
     func buildRelationships(_ relationships: Set<SymbolGraph.Relationship>, bundle: DocumentationBundle, engine: DiagnosticEngine) {
         for edge in relationships {
-            // Build conformant type <-> protocol relationships
-            if case .conformsTo = edge.kind {
+            switch edge.kind {
+            case .conformsTo:
+                // Build conformant type <-> protocol relationships
                 SymbolGraphRelationshipsBuilder.addConformanceRelationship(edge: edge, in: bundle, symbolIndex: &symbolIndex, engine: diagnosticEngine)
-                continue
-            }
-            
-            // Build implementation <-> protocol requirement relationships.
-            if case .defaultImplementationOf = edge.kind {
+            case .defaultImplementationOf:
+                // Build implementation <-> protocol requirement relationships.
                 SymbolGraphRelationshipsBuilder.addImplementationRelationship(edge: edge, in: bundle, context: self, symbolIndex: &symbolIndex, engine: diagnosticEngine)
-                continue
-            }
-            
-            // Build ancestor <-> offspring relationships.
-            if case .inheritsFrom = edge.kind {
+            case .inheritsFrom:
+                // Build ancestor <-> offspring relationships.
                 SymbolGraphRelationshipsBuilder.addInheritanceRelationship(edge: edge, in: bundle, symbolIndex: &symbolIndex, engine: diagnosticEngine)
-                continue
-            }
-            
-            // Build required member -> protocol relationships.
-            if case .requirementOf = edge.kind {
+            case .requirementOf:
+                // Build required member -> protocol relationships.
                 SymbolGraphRelationshipsBuilder.addRequirementRelationship(edge: edge, in: bundle, symbolIndex: &symbolIndex, engine: diagnosticEngine)
-                continue
+            case .optionalRequirementOf:
+                // Build optional required member -> protocol relationships.
+                SymbolGraphRelationshipsBuilder.addOptionalRequirementRelationship(edge: edge, in: bundle, symbolIndex: &symbolIndex, engine: diagnosticEngine)
+            default:
+                break
             }
         }
     }
@@ -1881,7 +1884,10 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
         let reference = ResolvedTopicReference(
             bundleIdentifier: bundle.identifier,
             path: path,
-            sourceLanguages: availableSourceLanguages ?? [.swift]
+            sourceLanguages: availableSourceLanguages
+                // FIXME: Pages in article-only catalogs should not be inferred as "Swift" as a fallback
+                // (github.com/apple/swift-docc/issues/240).
+                ?? [.swift]
         )
 
         let title = article.topicGraphNode.title

Sources/SwiftDocC/Infrastructure/DocumentationConverter.swift

@@ -311,8 +311,7 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
                         }
                     }
                 } catch {
-                    results.append(.init(description: error.localizedDescription, source: source))
-                    diagnosticEngine.emit(.init(description: error.localizedDescription, source: source))
+                    recordProblem(from: error, in: &results, withIdentifier: "render-node")
                 }
             }
         }
@@ -327,8 +326,7 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
         do {
             try outputConsumer.consume(assetsInBundle: bundle)
         } catch {
-            conversionProblems.append(.init(description: error.localizedDescription, source: nil))
-            diagnosticEngine.emit(.init(description: error.localizedDescription, source: nil))
+            recordProblem(from: error, in: &conversionProblems, withIdentifier: "assets")
         }
 
         // Write various metadata
@@ -338,17 +336,15 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
                 try outputConsumer.consume(indexingRecords: indexingRecords)
                 try outputConsumer.consume(assets: assets)
             } catch {
-                conversionProblems.append(.init(description: error.localizedDescription, source: nil))
-                diagnosticEngine.emit(.init(description: error.localizedDescription, source: nil))
+                recordProblem(from: error, in: &conversionProblems, withIdentifier: "metadata")
             }
         }
 
         if emitDigest {
             do {
                 try outputConsumer.consume(problems: context.problems + conversionProblems)
             } catch {
-                conversionProblems.append(.init(description: error.localizedDescription, source: nil))
-                diagnosticEngine.emit(.init(description: error.localizedDescription, source: nil))
+                recordProblem(from: error, in: &conversionProblems, withIdentifier: "problems")
             }
         }
 
@@ -357,9 +353,7 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
             do {
                 try outputConsumer.consume(documentationCoverageInfo: coverageInfo)
             } catch {
-                let problem = Problem(description: error.localizedDescription, source: nil)
-                conversionProblems.append(problem)
-                diagnosticEngine.emit(problem)
+                recordProblem(from: error, in: &conversionProblems, withIdentifier: "coverage")
             }
         case .none:
             break
@@ -412,6 +406,34 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
         return isDocumentPathToConvert || isExternalIDToConvert
     }
 
+    /// Record a problem from the given error in the given problem array.
+    ///
+    /// Creates a ``Problem`` from the given `Error` and identifier, emits it to the
+    /// ``DocumentationConverter``'s ``DiagnosticEngine``, and appends it to the given
+    /// problem array.
+    ///
+    /// - Parameters:
+    ///   - error: The error that describes the problem.
+    ///   - problems: The array that the created problem should be appended to.
+    ///   - identifier: A unique identifier the problem.
+    private func recordProblem(
+        from error: Swift.Error,
+        in problems: inout [Problem],
+        withIdentifier identifier: String
+    ) {
+        let singleDiagnostic = Diagnostic(
+            source: nil,
+            severity: .error,
+            range: nil,
+            identifier: "org.swift.docc.documentation-converter.\(identifier)",
+            summary: error.localizedDescription
+        )
+        let problem = Problem(diagnostic: singleDiagnostic, possibleSolutions: [])
+        
+        diagnosticEngine.emit(problem)
+        problems.append(problem)
+    }
+    
     enum Error: DescribedError {
         case doesNotContainBundle(url: URL)
 
@@ -427,21 +449,3 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
         }
     }
 }
-
-private extension Problem {
-    /// Creates a new problem with the given description and documentation source location.
-    ///
-    /// Use this to provide a user-friendly description of an error,
-    /// along with a direct reference to the source file and line number where you call this initializer.
-    ///
-    /// - Parameters:
-    ///   - description: A brief description of the problem.
-    ///   - source: The URL for the documentation file that caused this problem, if any.
-    ///   - file: The source file where you call this initializer.
-    init(description: String, source: URL?, file: String = #file) {
-        let fileName = URL(fileURLWithPath: file).deletingPathExtension().lastPathComponent
-        
-        let singleDiagnostic = Diagnostic(source: source, severity: .error, range: nil, identifier: "org.swift.docc.\(fileName)", summary: description)
-        self.init(diagnostic: singleDiagnostic, possibleSolutions: [])
-    }
-}

Sources/SwiftDocC/Infrastructure/Symbol Graph/SymbolGraphRelationshipsBuilder.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2022 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -232,22 +232,42 @@ struct SymbolGraphRelationshipsBuilder {
         }
     }
 
-    /// Adds a relationship from a type member to a protocol requirement.
+    /// Adds a required relationship from a type member to a protocol requirement.
     /// - Parameters:
     ///   - edge: A symbol graph relationship with a source and a target.
     ///   - bundle: A documentation bundle.
     ///   - symbolIndex: A symbol lookup map by precise identifier.
     ///   - engine: A diagnostic collecting engine.
     static func addRequirementRelationship(edge: SymbolGraph.Relationship, in bundle: DocumentationBundle, symbolIndex: inout [String: DocumentationNode], engine: DiagnosticEngine) {
+        addProtocolRelationship(edge: edge, in: bundle, symbolIndex: &symbolIndex, engine: engine, required: true)
+    }
+    
+    /// Adds an optional relationship from a type member to a protocol requirement.
+    /// - Parameters:
+    ///   - edge: A symbol graph relationship with a source and a target.
+    ///   - bundle: A documentation bundle.
+    ///   - symbolIndex: A symbol lookup map by precise identifier.
+    ///   - engine: A diagnostic collecting engine.
+    static func addOptionalRequirementRelationship(edge: SymbolGraph.Relationship, in bundle: DocumentationBundle, symbolIndex: inout [String: DocumentationNode], engine: DiagnosticEngine) {
+        addProtocolRelationship(edge: edge, in: bundle, symbolIndex: &symbolIndex, engine: engine, required: false)
+    }
+    
+    /// Adds a relationship from a type member to a protocol requirement.
+    /// - Parameters:
+    ///   - edge: A symbol graph relationship with a source and a target.
+    ///   - bundle: A documentation bundle.
+    ///   - symbolIndex: A symbol lookup map by precise identifier.
+    ///   - engine: A diagnostic collecting engine.
+    ///   - required: A bool value indicating whether the protocol requirement is required or optional
+    private static func addProtocolRelationship(edge: SymbolGraph.Relationship, in bundle: DocumentationBundle, symbolIndex: inout [String: DocumentationNode], engine: DiagnosticEngine, required: Bool) {
         // Resolve source symbol
         guard let requiredNode = symbolIndex[edge.source],
             let requiredSymbol = requiredNode.semantic as? Symbol else {
             // The source node for requirement relationship not found.
             engine.emit(NodeProblem.notFound(edge.source))
             return
         }
-        
-        requiredSymbol.isRequired = true
+        requiredSymbol.isRequired = required
     }
 
     /// Sets a node in the context as an inherited symbol if the origin symbol is provided in the given relationship.

Sources/SwiftDocC/Model/Identifier.swift

@@ -96,6 +96,15 @@ public struct ResolvedTopicReference: Hashable, Codable, Equatable, CustomString
     static func purgePool(for bundleIdentifier: String) {
         sharedPool.sync { $0.removeValue(forKey: bundleIdentifier) }
     }
+    
+    /// Enables reference caching for any identifiers created with the given bundle identifier.
+    static func enableReferenceCaching(for bundleIdentifier: ReferenceBundleIdentifier) {
+        sharedPool.sync { sharedPool in
+            if !sharedPool.keys.contains(bundleIdentifier) {
+                sharedPool[bundleIdentifier] = [:]
+            }
+        }
+    }
 
     /// The URL scheme for `doc://` links.
     public static let urlScheme = "doc"
@@ -174,7 +183,10 @@ public struct ResolvedTopicReference: Hashable, Codable, Equatable, CustomString
         )
 
         // Cache the reference
-        Self.sharedPool.sync { $0[bundleIdentifier, default: [:]][key] = self }
+        Self.sharedPool.sync { sharedPool in
+            // If we have a shared pool for this bundle identifier, cache the reference
+            sharedPool[bundleIdentifier]?[key] = self
+        }
     }
 
     private static func cacheKey(