Add experimental code block options: highlight, strikeout, wrap, showLineNumbers (#1287)

* update docs

* copy by default

* add feature flag 'enable-experimental-code-block-annotations' for copy-to-clipboard and other code block annotations

* add more tests, remove docs, code cleanup

* add code block options onto RenderBlockContent.CodeListing

* remaining PR feedback

* copy by default

* add feature flag 'enable-experimental-code-block-annotations' for copy-to-clipboard and other code block annotations

* WIP wrap and highlight

* fix tests

* WIP tests, WIP parsing for wrap and highlight

* change parsing to handle values after = and arrays

* add strikeout option

* parse strikeout option, solution for language not as the first option on language line, tests

* validate array values in code block options for highlight and strikeout

* showLineNumbers option

* remove trailing comma

* test showLineNumbers

* PR feedback

* fix feature flag on new tests

* remove optional return type

* update JSON structure for extensibility

* update RenderNode.spec to reflect using Range<Position> in LineAnnotation

* update feature name

* Update Sources/SwiftDocC/Model/Rendering/Content/RenderBlockContent.swift

Co-authored-by: David Rönnqvist <[email protected]>

* Update Sources/SwiftDocC/Model/Rendering/Content/RenderBlockContent.swift

Co-authored-by: David Rönnqvist <[email protected]>

* require LineAnnotation properties style and range

* fix CodeListing initializers in Snippet

* fix typo

* add copy-to-clipboard button back to snippets when feature flag is present

---------

Co-authored-by: Jesse Haigh <[email protected]>
Co-authored-by: David Rönnqvist <[email protected]>

Author: 3 people

Sources/SwiftDocC/Checker/Checkers/InvalidCodeBlockOption.swift

@@ -19,7 +19,7 @@ internal struct InvalidCodeBlockOption: Checker {
     var problems = [Problem]()
 
     /// Parsing options for code blocks
-    private let knownOptions = RenderBlockContent.CodeListing.knownOptions
+    private let knownOptions = RenderBlockContent.CodeBlockOptions.knownOptions
 
     private var sourceFile: URL?
 
@@ -31,32 +31,67 @@ internal struct InvalidCodeBlockOption: Checker {
     }
 
     mutating func visitCodeBlock(_ codeBlock: CodeBlock) {
-        let info = codeBlock.language?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
-        guard !info.isEmpty else { return }
+        let (lang, tokens) = RenderBlockContent.CodeBlockOptions.tokenizeLanguageString(codeBlock.language)
 
-        let tokens = info
-            .split(separator: ",")
-            .map { $0.trimmingCharacters(in: .whitespaces) }
-            .filter { !$0.isEmpty }
+        func matches(token: RenderBlockContent.CodeBlockOptions.OptionName, value: String?) {
+            guard token == .unknown, let value = value else { return }
 
-        guard !tokens.isEmpty else { return }
-
-        for token in tokens {
-            // if the token is an exact match, we don't need to do anything
-            guard !knownOptions.contains(token) else { continue }
-
-            let matches = NearMiss.bestMatches(for: knownOptions, against: token)
+            let matches = NearMiss.bestMatches(for: knownOptions, against: value)
 
             if !matches.isEmpty {
-                let diagnostic = Diagnostic(source: sourceFile, severity: .warning, range: codeBlock.range, identifier: "org.swift.docc.InvalidCodeBlockOption", summary: "Unknown option \(token.singleQuoted) in code block.")
+                let diagnostic = Diagnostic(source: sourceFile, severity: .warning, range: codeBlock.range, identifier: "org.swift.docc.InvalidCodeBlockOption", summary: "Unknown option \(value.singleQuoted) in code block.")
                 let possibleSolutions = matches.map { candidate in
                     Solution(
-                        summary: "Replace \(token.singleQuoted) with \(candidate.singleQuoted).",
+                        summary: "Replace \(value.singleQuoted) with \(candidate.singleQuoted).",
                         replacements: []
                     )
                 }
                 problems.append(Problem(diagnostic: diagnostic, possibleSolutions: possibleSolutions))
+            } else if lang == nil {
+                let diagnostic = Diagnostic(source: sourceFile, severity: .warning, range: codeBlock.range, identifier: "org.swift.docc.InvalidCodeBlockOption", summary: "Unknown option \(value.singleQuoted) in code block.")
+                let possibleSolutions =
+                Solution(
+                    summary: "If \(value.singleQuoted) is the language for this code block, then write \(value.singleQuoted) as the first option.",
+                    replacements: []
+                )
+                problems.append(Problem(diagnostic: diagnostic, possibleSolutions: [possibleSolutions]))
+            }
+        }
+
+        func validateArrayIndices(token: RenderBlockContent.CodeBlockOptions.OptionName, value: String?) {
+            guard token == .highlight || token == .strikeout, let value = value else { return }
+            // code property ends in a newline. this gives us a bogus extra line.
+            let lineCount: Int = codeBlock.code.split(omittingEmptySubsequences: false, whereSeparator: { $0.isNewline }).count - 1
+
+            let indices = RenderBlockContent.CodeBlockOptions.parseCodeBlockOptionsArray(value)
+
+            if !value.isEmpty, indices.isEmpty {
+                let diagnostic = Diagnostic(source: sourceFile, severity: .warning, range: codeBlock.range, identifier: "org.swift.docc.InvalidCodeBlockOption", summary: "Could not parse \(token.rawValue.singleQuoted) indices from \(value.singleQuoted). Expected an integer (e.g. 3) or an array (e.g. [1, 3, 5])")
+                problems.append(Problem(diagnostic: diagnostic, possibleSolutions: []))
+                return
             }
+
+            let invalid = indices.filter { $0 < 1 || $0 > lineCount }
+            guard !invalid.isEmpty else { return }
+
+            let diagnostic = Diagnostic(source: sourceFile, severity: .warning, range: codeBlock.range, identifier: "org.swift.docc.InvalidCodeBlockOption", summary: "Invalid \(token.rawValue.singleQuoted) index\(invalid.count == 1 ? "" : "es") in \(value.singleQuoted) for a code block with \(lineCount) line\(lineCount == 1 ? "" : "s"). Valid range is 1...\(lineCount).")
+            let solutions: [Solution] = {
+                if invalid.contains(where: {$0 == lineCount + 1}) {
+                    return [Solution(
+                        summary: "If you intended the last line, change '\(lineCount + 1)' to \(lineCount).",
+                        replacements: []
+                    )]
+                }
+                return []
+            }()
+            problems.append(Problem(diagnostic: diagnostic, possibleSolutions: solutions))
+        }
+
+        for (token, value) in tokens {
+            matches(token: token, value: value)
+            validateArrayIndices(token: token, value: value)
         }
+        // check if first token (lang) might be a typo
+        matches(token: .unknown, value: lang)
     }
 }

Sources/SwiftDocC/Model/Rendering/Content/RenderBlockContent.swift

@@ -124,10 +124,60 @@ public enum RenderBlockContent: Equatable {
         public var code: [String]
         /// Additional metadata for this code block.
         public var metadata: RenderContentMetadata?
+        /// Annotations for code blocks
+        public var options: CodeBlockOptions?
+
+        /// Make a new `CodeListing` with the given data.
+        public init(syntax: String?, code: [String], metadata: RenderContentMetadata?, options: CodeBlockOptions?) {
+            self.syntax = syntax
+            self.code = code
+            self.metadata = metadata
+            self.options = options
+        }
+    }
+
+    public struct CodeBlockOptions: Equatable {
+        public var language: String?
         public var copyToClipboard: Bool
+        public var showLineNumbers: Bool
+        public var wrap: Int
+        public var lineAnnotations: [LineAnnotation]
+
+        public struct Position: Equatable, Comparable, Codable {
+            public static func < (lhs: RenderBlockContent.CodeBlockOptions.Position, rhs: RenderBlockContent.CodeBlockOptions.Position) -> Bool {
+                if lhs.line == rhs.line, let lhsCharacter = lhs.character, let rhsCharacter = rhs.character {
+                    return lhsCharacter < rhsCharacter
+                }
+                return lhs.line < rhs.line
+            }
+
+            public init(line: Int, character: Int? = nil) {
+                self.line = line
+                self.character = character
+            }
+
+            public var line: Int
+            public var character: Int?
+        }
+
+        public struct LineAnnotation: Equatable, Codable {
+            public var style: String
+            public var range: Range<Position>
+            
+            public init(style: String, range: Range<Position>) {
+                self.style = style
+                self.range = range
+            }
+        }
 
         public enum OptionName: String, CaseIterable {
+            case _nonFrozenEnum_useDefaultCase
             case nocopy
+            case wrap
+            case highlight
+            case showLineNumbers
+            case strikeout
+            case unknown
 
             init?(caseInsensitive raw: some StringProtocol) {
                 self.init(rawValue: raw.lowercased())
@@ -138,12 +188,165 @@ public enum RenderBlockContent: Equatable {
             Set(OptionName.allCases.map(\.rawValue))
         }
 
-        /// Make a new `CodeListing` with the given data.
-        public init(syntax: String?, code: [String], metadata: RenderContentMetadata?, copyToClipboard: Bool = FeatureFlags.current.isExperimentalCodeBlockAnnotationsEnabled) {
-            self.syntax = syntax
-            self.code = code
-            self.metadata = metadata
+        // empty initializer with default values
+        public init() {
+            self.language = ""
+            self.copyToClipboard = FeatureFlags.current.isExperimentalCodeBlockAnnotationsEnabled
+            self.showLineNumbers = false
+            self.wrap = 0
+            self.lineAnnotations = []
+        }
+
+        public init(parsingLanguageString language: String?) {
+            let (lang, tokens) = Self.tokenizeLanguageString(language)
+
+            self.language = lang
+            self.copyToClipboard = !tokens.contains { $0.name == .nocopy }
+            self.showLineNumbers = tokens.contains { $0.name == .showLineNumbers }
+
+            if let wrapString = tokens.first(where: { $0.name == .wrap })?.value,
+               let wrapValue = Int(wrapString) {
+                self.wrap = wrapValue
+            } else {
+                self.wrap = 0
+            }
+
+            var annotations: [LineAnnotation] = []
+
+            if let highlightString = tokens.first(where: { $0.name == .highlight })?.value {
+                let highlightValue = Self.parseCodeBlockOptionsArray(highlightString)
+                for line in highlightValue {
+                    let pos = Position(line: line, character: nil)
+                    let range = pos..<pos
+                    annotations.append(LineAnnotation(style: "highlight", range: range))
+                }
+            }
+
+            if let strikeoutString = tokens.first(where: { $0.name == .strikeout })?.value {
+                let strikeoutValue = Self.parseCodeBlockOptionsArray(strikeoutString)
+                for line in strikeoutValue {
+                    let pos = Position(line: line, character: nil)
+                    let range = pos..<pos
+                    annotations.append(LineAnnotation(style: "strikeout", range: range))
+                }
+            }
+
+            self.lineAnnotations = annotations
+        }
+
+        public init(copyToClipboard: Bool = FeatureFlags.current.isExperimentalCodeBlockAnnotationsEnabled, showLineNumbers: Bool = false, wrap: Int, highlight: [Int], strikeout: [Int]) {
+            self.copyToClipboard = copyToClipboard
+            self.showLineNumbers = showLineNumbers
+            self.wrap = wrap
+
+            var annotations: [LineAnnotation] = []
+            for line in highlight {
+                    let pos = Position(line: line, character: nil)
+                    let range = pos..<pos
+                    annotations.append(LineAnnotation(style: "highlight", range: range))
+            }
+            for line in strikeout {
+                    let pos = Position(line: line, character: nil)
+                    let range = pos..<pos
+                    annotations.append(LineAnnotation(style: "strikeout", range: range))
+            }
+            self.lineAnnotations = annotations
+        }
+
+        public init(copyToClipboard: Bool, showLineNumbers: Bool, wrap: Int, lineAnnotations: [LineAnnotation]) {
             self.copyToClipboard = copyToClipboard
+            self.showLineNumbers = showLineNumbers
+            self.wrap = wrap
+            self.lineAnnotations = lineAnnotations
+        }
+
+        /// A function that parses array values on code block options from the language line string
+        static internal func parseCodeBlockOptionsArray(_ value: String?) -> [Int] {
+            guard var s = value?.trimmingCharacters(in: .whitespaces), !s.isEmpty else { return [] }
+
+            if s.hasPrefix("[") && s.hasSuffix("]") {
+                s.removeFirst()
+                s.removeLast()
+            }
+
+            return s.split(separator: ",").compactMap { Int($0.trimmingCharacters(in: .whitespaces)) }
+        }
+
+        /// A function that parses the language line options on code blocks, returning the language and tokens, an array of OptionName and option values
+        static internal func tokenizeLanguageString(_ input: String?) -> (lang: String?, tokens: [(name: OptionName, value: String?)]) {
+            guard let input else { return (lang: nil, tokens: []) }
+
+            let parts = parseLanguageString(input)
+            var tokens: [(OptionName, String?)] = []
+            var lang: String? = nil
+
+            for (index, part) in parts.enumerated() {
+                if let eq = part.firstIndex(of: "=") {
+                    let key = part[..<eq].trimmingCharacters(in: .whitespaces).lowercased()
+                    let value = part[part.index(after: eq)...].trimmingCharacters(in: .whitespaces)
+                    if key == "wrap" {
+                        tokens.append((.wrap, value))
+                    } else if key == "highlight" {
+                        tokens.append((.highlight, value))
+                    } else if key == "strikeout" {
+                        tokens.append((.strikeout, value))
+                    } else {
+                        tokens.append((.unknown, key))
+                    }
+                } else {
+                    let key = part.trimmingCharacters(in: .whitespaces).lowercased()
+                    if key == "nocopy" {
+                        tokens.append((.nocopy, nil as String?))
+                    } else if key == "showlinenumbers" {
+                        tokens.append((.showLineNumbers, nil as String?))
+                    } else if key == "wrap" {
+                        tokens.append((.wrap, nil as String?))
+                    } else if key == "highlight" {
+                        tokens.append((.highlight, nil as String?))
+                    } else if key == "strikeout" {
+                        tokens.append((.strikeout, nil as String?))
+                    } else if index == 0 && !key.contains("[") && !key.contains("]") {
+                        lang = key
+                    } else {
+                        tokens.append((.unknown, key))
+                    }
+                }
+            }
+            return (lang, tokens)
+        }
+
+        // helper function for tokenizeLanguageString to parse the language line
+        static func parseLanguageString(_ input: String?) -> [Substring] {
+
+            guard let input else { return [] }
+            var parts: [Substring] = []
+            var start = input.startIndex
+            var i = input.startIndex
+
+            var bracketDepth = 0
+
+            while i < input.endIndex {
+                let c = input[i]
+
+                if c == "[" { bracketDepth += 1 }
+                else if c == "]" { bracketDepth = max(0, bracketDepth - 1) }
+                else if c == "," && bracketDepth == 0 {
+                    let seq = input[start..<i]
+                    if !seq.isEmpty {
+                        parts.append(seq)
+                    }
+                    input.formIndex(after: &i)
+                    start = i
+                    continue
+                }
+                input.formIndex(after: &i)
+            }
+            let tail = input[start..<input.endIndex]
+            if !tail.isEmpty {
+                parts.append(tail)
+            }
+
+            return parts
         }
     }
 
@@ -711,7 +914,7 @@ extension RenderBlockContent.Table: Codable {
 extension RenderBlockContent: Codable {
     private enum CodingKeys: CodingKey {
         case type
-        case inlineContent, content, caption, style, name, syntax, code, level, text, items, media, runtimePreview, anchor, summary, example, metadata, start, copyToClipboard
+        case inlineContent, content, caption, style, name, syntax, code, level, text, items, media, runtimePreview, anchor, summary, example, metadata, start, copyToClipboard, showLineNumbers, wrap, lineAnnotations
         case request, response
         case header, rows
         case numberOfColumns, columns
@@ -734,12 +937,23 @@ extension RenderBlockContent: Codable {
             self = try .aside(.init(style: style, content: container.decode([RenderBlockContent].self, forKey: .content)))
         case .codeListing:
             let copy = FeatureFlags.current.isExperimentalCodeBlockAnnotationsEnabled
+            let options: CodeBlockOptions?
+            if !Set(container.allKeys).isDisjoint(with: [.copyToClipboard, .showLineNumbers, .wrap, .lineAnnotations]) {
+                options = try CodeBlockOptions(
+                    copyToClipboard: container.decodeIfPresent(Bool.self, forKey: .copyToClipboard) ?? copy,
+                    showLineNumbers: container.decodeIfPresent(Bool.self, forKey: .showLineNumbers) ?? false,
+                    wrap: container.decodeIfPresent(Int.self, forKey: .wrap) ?? 0,
+                    lineAnnotations: container.decodeIfPresent([CodeBlockOptions.LineAnnotation].self, forKey: .lineAnnotations) ?? []
+                )
+            } else {
+                options = nil
+            }
             self = try .codeListing(.init(
                 syntax: container.decodeIfPresent(String.self, forKey: .syntax),
                 code: container.decode([String].self, forKey: .code),
                 metadata: container.decodeIfPresent(RenderContentMetadata.self, forKey: .metadata),
-                copyToClipboard: container.decodeIfPresent(Bool.self, forKey: .copyToClipboard) ?? copy
-                ))
+                options: options
+            ))
         case .heading:
             self = try .heading(.init(level: container.decode(Int.self, forKey: .level), text: container.decode(String.self, forKey: .text), anchor: container.decodeIfPresent(String.self, forKey: .anchor)))
         case .orderedList:
@@ -842,7 +1056,10 @@ extension RenderBlockContent: Codable {
             try container.encode(l.syntax, forKey: .syntax)
             try container.encode(l.code, forKey: .code)
             try container.encodeIfPresent(l.metadata, forKey: .metadata)
-            try container.encode(l.copyToClipboard, forKey: .copyToClipboard)
+            try container.encodeIfPresent(l.options?.copyToClipboard, forKey: .copyToClipboard)
+            try container.encodeIfPresent(l.options?.showLineNumbers, forKey: .showLineNumbers)
+            try container.encodeIfPresent(l.options?.wrap, forKey: .wrap)
+            try container.encodeIfPresent(l.options?.lineAnnotations, forKey: .lineAnnotations)
         case .heading(let h):
             try container.encode(h.level, forKey: .level)
             try container.encode(h.text, forKey: .text)