Preserve HTML entities in headers

Closes #2114.
Closes #2120.

Author: josevalim

lib/ex_doc/doc_ast.ex

@@ -110,6 +110,18 @@ defmodule ExDoc.DocAST do
   def extract_title([{:h1, _attrs, inner, _meta} | ast]), do: {:ok, inner, ast}
   def extract_title(_ast), do: :error
 
+  @doc """
+  Extracts headers (h2) from the given AST.
+
+  Returns the header text.
+  """
+  def extract_headers(doc_ast) do
+    for {:h2, _, _, _} = node <- doc_ast,
+        text = ExDoc.DocAST.text(node),
+        text != "",
+        do: text
+  end
+
   @doc """
   Compute a synopsis from a document by looking at its first paragraph.
   """

lib/ex_doc/formatter/html.ex

@@ -320,7 +320,10 @@ defmodule ExDoc.Formatter.HTML do
       source_path: nil,
       source_url: config.source_url,
       title: "API Reference",
-      title_content: title_content
+      title_content: title_content,
+      headers:
+        if(nodes_map.modules != [], do: ["Modules"], else: []) ++
+          if(nodes_map.tasks != [], do: ["Mix Tasks"], else: [])
     }
   end
 
@@ -456,7 +459,8 @@ defmodule ExDoc.Formatter.HTML do
       source_url: source_url,
       search_data: search_data,
       title: title,
-      title_content: title_html || title
+      title_content: title_html || title,
+      headers: ExDoc.DocAST.extract_headers(ast)
     }
   end
 

lib/ex_doc/formatter/html/templates.ex

@@ -59,7 +59,6 @@ defmodule ExDoc.Formatter.HTML.Templates do
   defp sidebar_extras(extras) do
     for extra <- extras do
       %{id: id, title: title, group: group} = extra
-
       item = %{id: to_string(id), title: to_string(title), group: to_string(group)}
 
       case extra do
@@ -74,14 +73,14 @@ defmodule ExDoc.Formatter.HTML.Templates do
             end)
 
           item
-          |> Map.put(:headers, extract_headers(extra.content))
+          |> Map.put(:headers, headers_to_id_and_anchors(extra.headers))
           |> Map.put(:searchData, search_data)
 
         %{url: url} when is_binary(url) ->
           Map.put(item, :url, url)
 
         _ ->
-          Map.put(item, :headers, extract_headers(extra.content))
+          Map.put(item, :headers, headers_to_id_and_anchors(extra.headers))
       end
     end
   end
@@ -140,8 +139,9 @@ defmodule ExDoc.Formatter.HTML.Templates do
 
   defp module_sections(module) do
     {sections, _} =
-      module.rendered_doc
-      |> extract_headers()
+      module.doc
+      |> ExDoc.DocAST.extract_headers()
+      |> headers_to_id_and_anchors()
       |> Enum.map_reduce(%{}, fn header, acc ->
         # TODO Duplicates some of the logic of link_headings/3
         case Map.fetch(acc, header.id) do
@@ -156,14 +156,10 @@ defmodule ExDoc.Formatter.HTML.Templates do
     [sections: sections]
   end
 
-  # TODO: split into sections in Formatter.HTML instead (possibly via DocAST)
-  defp extract_headers(content) do
-    ~r/<h2.*?>(.*?)<\/h2>/m
-    |> Regex.scan(content, capture: :all_but_first)
-    |> List.flatten()
-    |> Enum.filter(&(&1 != ""))
-    |> Enum.map(&ExDoc.Utils.strip_tags/1)
-    |> Enum.map(&%{id: &1, anchor: URI.encode(text_to_id(&1))})
+  defp headers_to_id_and_anchors(headers) do
+    Enum.map(headers, fn text ->
+      %{id: text, anchor: URI.encode(text_to_id(text))}
+    end)
   end
 
   def module_summary(module_node) do

test/ex_doc/doc_ast_test.exs

@@ -150,6 +150,31 @@ defmodule ExDoc.DocASTTest do
     end
   end
 
+  describe "extract_headers" do
+    test "extracts h2 headers" do
+      assert extract_headers("""
+             # h1
+             ## h2-a
+             ### h3
+             ## h2-b
+             ##
+             """) == ["h2-a", "h2-b"]
+    end
+
+    test "trims whitespace and preserve HTML entities" do
+      assert extract_headers("""
+             # h1
+             ##\s\s\sh2\s<&>\sh2\s\s\s
+             """) == ["h2 <&> h2"]
+    end
+
+    defp extract_headers(markdown) do
+      markdown
+      |> ExDoc.DocAST.parse!("text/markdown")
+      |> ExDoc.DocAST.extract_headers()
+    end
+  end
+
   describe "highlight" do
     test "with default class" do
       # Empty class

test/ex_doc/formatter/html/templates_test.exs

@@ -324,50 +324,6 @@ defmodule ExDoc.Formatter.HTML.TemplatesTest do
              ] = create_sidebar_items(%{modules: nodes}, [])["modules"]
     end
 
-    test "outputs extras with headers" do
-      item = %{content: nil, group: nil, id: nil, title: nil}
-
-      assert create_sidebar_items(%{}, [%{item | content: "<h2>Foo</h2><h2>Bar</h2>"}])["extras"] ==
-               [
-                 %{
-                   "group" => "",
-                   "headers" => [
-                     %{"anchor" => "foo", "id" => "Foo"},
-                     %{"anchor" => "bar", "id" => "Bar"}
-                   ],
-                   "id" => "",
-                   "title" => ""
-                 }
-               ]
-
-      assert create_sidebar_items(%{}, [%{item | content: "<h2>Foo</h2>\n<h2>Bar</h2>"}])[
-               "extras"
-             ] ==
-               [
-                 %{
-                   "group" => "",
-                   "headers" => [
-                     %{"anchor" => "foo", "id" => "Foo"},
-                     %{"anchor" => "bar", "id" => "Bar"}
-                   ],
-                   "id" => "",
-                   "title" => ""
-                 }
-               ]
-
-      assert create_sidebar_items(%{}, [%{item | content: "<h2>Foo</h2><h2></h2>"}])["extras"] ==
-               [
-                 %{
-                   "group" => "",
-                   "headers" => [
-                     %{"anchor" => "foo", "id" => "Foo"}
-                   ],
-                   "id" => "",
-                   "title" => ""
-                 }
-               ]
-    end
-
     test "builds sections out of moduledocs", context do
       names = [CompiledWithDocs, CompiledWithoutDocs, DuplicateHeadings]
       config = doc_config(context)

test/ex_doc/formatter/html_test.exs

@@ -617,7 +617,7 @@ defmodule ExDoc.Formatter.HTMLTest do
                  "headers" => [
                    %{"anchor" => "heading-without-content", "id" => "Heading without content"},
                    %{"anchor" => "header-sample", "id" => "Header sample"},
-                   %{"anchor" => "more-than", "id" => "more > than"}
+                   %{"anchor" => "more-than", "id" => "more > than"}
                  ]
                },
                %{"id" => "livebookfile"},
@@ -768,7 +768,7 @@ defmodule ExDoc.Formatter.HTMLTest do
                  "headers" => [
                    %{"anchor" => "heading-without-content", "id" => "Heading without content"},
                    %{"anchor" => "header-sample", "id" => "Header sample"},
-                   %{"anchor" => "more-than", "id" => "more > than"}
+                   %{"anchor" => "more-than", "id" => "more > than"}
                  ]
                }
              ] = Jason.decode!(content)["extras"]