✨ NEW: Add linkify

.github/workflows/tests.yml

@@ -40,7 +40,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install .[testing]
+        pip install .[testing,linkify]
     - name: Run pytest
       run: |
         pytest --cov=markdown_it --cov-report=xml --cov-report=term-missing

markdown_it/main.py

@@ -11,6 +11,11 @@
 from .renderer import RendererHTML
 from .utils import AttrDict
 
+try:
+    import linkify_it
+except ModuleNotFoundError:
+    linkify_it = None
+
 
 _PRESETS = AttrDict(
     {
@@ -41,8 +46,7 @@ def __init__(
         self.options = {}
         self.configure(config)
 
-        # var LinkifyIt    = require('linkify-it')
-        # self.linkify = LinkifyIt()  # TODO maybe see https://github.com/Suor/autolink
+        self.linkify = linkify_it.LinkifyIt() if linkify_it else None
 
     def __repr__(self):
         return f"{self.__class__.__module__}.{self.__class__.__name__}()"

markdown_it/parser_core.py

@@ -8,14 +8,14 @@
 
 from .ruler import Ruler
 from .rules_core.state_core import StateCore
-from .rules_core import normalize, block, inline, replace, smartquotes
+from .rules_core import normalize, block, inline, replace, smartquotes, linkify
+
 
-# TODO linkify
 _rules = [
     ["normalize", normalize],
     ["block", block],
     ["inline", inline],
-    #   [ 'linkify',        require('./rules_core/linkify')        ],
+    ["linkify", linkify],
     ["replacements", replace],
     ["smartquotes", smartquotes],
 ]

markdown_it/rules_core/__init__.py

@@ -4,3 +4,4 @@
 from .inline import inline  # noqa: F401
 from .replacements import replace  # noqa: F401
 from .smartquotes import smartquotes  # noqa: F401
+from .linkify import linkify  # noqa: F401

markdown_it/rules_core/linkify.py

@@ -0,0 +1,141 @@
+import re
+
+from ..common.utils import arrayReplaceAt
+from ..common.normalize_url import normalizeLinkText, normalizeLink, validateLink
+from .state_core import StateCore
+from ..token import Token
+
+
+LINK_OPEN_RE = re.compile(r"^<a[>\s]", flags=re.IGNORECASE)
+LINK_CLOSE_RE = re.compile(r"^</a\s*>", flags=re.IGNORECASE)
+
+HTTP_RE = re.compile(r"^http://")
+MAILTO_RE = re.compile(r"^mailto:")
+TEST_MAILTO_RE = re.compile(r"^mailto:", flags=re.IGNORECASE)
+
+
+def isLinkOpen(string: str):
+    return LINK_OPEN_RE.search(string)
+
+
+def isLinkClose(string: str):
+    return LINK_CLOSE_RE.search(string)
+
+
+def linkify(state: StateCore):
+    blockTokens = state.tokens
+
+    if not state.md.options.linkify:
+        return
+
+    if not state.md.linkify:
+        raise ModuleNotFoundError("Linkify enabled but not installed.")
+
+    for j in range(len(blockTokens)):
+        if blockTokens[j].type != "inline" or not state.md.linkify.pretest(
+            blockTokens[j].content
+        ):
+            continue
+
+        tokens = blockTokens[j].children
+
+        htmlLinkLevel = 0
+
+        # We scan from the end, to keep position when new tags added.
+        # Use reversed logic in links start/end match
+        assert tokens is not None
+        i = len(tokens)
+        while i >= 1:
+            i -= 1
+            assert isinstance(tokens, list)
+            currentToken = tokens[i]
+
+            # Skip content of markdown links
+            if currentToken.type == "link_close":
+                i -= 1
+                while (
+                    tokens[i].level != currentToken.level
+                    and tokens[i].type != "link_open"
+                ):
+                    i -= 1
+                continue
+
+            # Skip content of html tag links
+            if currentToken.type == "html_inline":
+                if isLinkOpen(currentToken.content) and htmlLinkLevel > 0:
+                    htmlLinkLevel -= 1
+                if isLinkClose(currentToken.content):
+                    htmlLinkLevel += 1
+            if htmlLinkLevel > 0:
+                continue
+
+            if currentToken.type == "text" and state.md.linkify.test(
+                currentToken.content
+            ):
+                text = currentToken.content
+                links = state.md.linkify.match(text)
+
+                # Now split string to nodes
+                nodes = []
+                level = currentToken.level
+                lastPos = 0
+
+                for ln in range(len(links)):
+                    url = links[ln].url
+                    fullUrl = normalizeLink(url)
+                    if not validateLink(fullUrl):
+                        continue
+
+                    urlText = links[ln].text
+
+                    # Linkifier might send raw hostnames like "example.com", where url
+                    # starts with domain name. So we prepend http:// in those cases,
+                    # and remove it afterwards.
+                    if not links[ln].schema:
+                        urlText = HTTP_RE.sub(
+                            "", normalizeLinkText("http://" + urlText)
+                        )
+                    elif links[ln].schema == "mailto:" and TEST_MAILTO_RE.search(
+                        urlText
+                    ):
+                        urlText = MAILTO_RE.sub(
+                            "", normalizeLinkText("mailto:" + urlText)
+                        )
+                    else:
+                        urlText = normalizeLinkText(urlText)
+
+                    pos = links[ln].index
+
+                    if pos > lastPos:
+                        token = Token("text", "", 0)
+                        token.content = text[lastPos:pos]
+                        token.level = level
+                        nodes.append(token)
+
+                    token = Token("link_open", "a", 1)
+                    token.attrs = [["href", fullUrl]]
+                    token.level = level + 1
+                    token.markup = "linkify"
+                    token.info = "auto"
+                    nodes.append(token)
+
+                    token = Token("text", "", 0)
+                    token.content = urlText
+                    token.level = level
+                    nodes.append(token)
+
+                    token = Token("link_close", "a", -1)
+                    token.level = level - 1
+                    token.markup = "linkify"
+                    token.info = "auto"
+                    nodes.append(token)
+
+                    lastPos = links[ln].last_index
+
+                if lastPos < len(text):
+                    token = Token("text", "", 0)
+                    token.content = text[lastPos:]
+                    token.level = level
+                    nodes.append(token)
+
+                blockTokens[j].children = tokens = arrayReplaceAt(tokens, i, nodes)

setup.py

@@ -67,6 +67,7 @@ def get_version():
             "mistletoe-ebp~=0.10.0",
             "panflute~=1.12",
         ],
+        "linkify": ["linkify-it-py~=1.0"],
     },
     zip_safe=False,
 )

tests/test_api/test_main.py

@@ -8,7 +8,14 @@ def test_get_rules():
     md = MarkdownIt("zero")
     # print(md.get_all_rules())
     assert md.get_all_rules() == {
-        "core": ["normalize", "block", "inline", "replacements", "smartquotes"],
+        "core": [
+            "normalize",
+            "block",
+            "inline",
+            "linkify",
+            "replacements",
+            "smartquotes",
+        ],
         "block": [
             "table",
             "code",