feat: docs can be built with both sphinx & mkdocs

- adds back sphinx configs
- refactors `docs/gen_docs.py` to generate both
sphinx and mkdocs based docs as needed
- ignores dynamically generated doc files
- updates development dependencies

**Related items**

*Issues*

- Closes #258

Author: yozachar

.gitignore

@@ -69,7 +69,9 @@ instance/
 .scrapy
 
 # Sphinx documentation
+docs/reference/
 docs/_build/
+docs/*.rst
 
 # PyBuilder
 .pybuilder/

build.py

@@ -10,7 +10,7 @@
 
 if __name__ == "__main__":
     project_dir = Path(__file__).parent
-    generate_documentation(project_dir, discard_refs=False)
+    generate_documentation(project_dir, only_md=True, discard_refs=False)
     process = run(("poetry", "build"), capture_output=True)
     print(process.stderr.decode() + process.stdout.decode())
     rmtree(project_dir / "docs/reference", ignore_errors=True)

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,41 @@
+"""Configuration file for the Sphinx documentation builder.
+
+For the full list of built-in configuration values, see the documentation:
+https://www.sphinx-doc.org/en/master/usage/configuration.html
+"""
+
+# standard
+from importlib.metadata import metadata
+from datetime import datetime
+
+# -- Project information ----------------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+_metadata = metadata("validators")
+
+project: str = _metadata["name"]
+author: str = _metadata["author"]
+project_copyright = f"2013 - {datetime.now().year}, {_metadata['author']}"
+version: str = _metadata["version"]
+release = version
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "myst_parser",
+]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "*.md"]
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+html_theme = "alabaster"
+
+# -- Options for manpage generation -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-man_pages
+man_pages = [("index", project, _metadata["summary"], [author], 1)]
+
+# -- Options for docstring parsing -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html
+napoleon_numpy_docstring = False

docs/gen_docs.py

@@ -8,6 +8,7 @@
 from os.path import getsize
 from subprocess import run
 from pathlib import Path
+from sys import argv
 
 # external
 from yaml import safe_load, safe_dump
@@ -19,28 +20,37 @@ def _write_ref_content(source: Path, module_name: str, func_name: str):
     """Write content."""
     with open(source, "at") as ref:
         ref.write(
-            (f"# {module_name}\n\n" if getsize(source) == 0 else "")
-            + f"::: validators.{module_name}.{func_name}\n"
+            (
+                (f"# {module_name}\n\n" if getsize(source) == 0 else "")
+                + f"::: validators.{module_name}.{func_name}\n"
+            )
+            if f"{source}".endswith(".md")
+            else (
+                (f"{module_name}\n{len(module_name) * '-'}\n\n" if getsize(source) == 0 else "")
+                + f".. module:: validators.{module_name}\n"
+                + f".. autofunction:: {func_name}\n"
+            )
         )
 
 
-def _generate_reference(source: Path, destination: Path):
-    """Generate reference."""
-    nav_items: Dict[str, List[str]] = {"Code Reference": []}
-    # clean destination
-    if destination.exists() and destination.is_dir():
-        rmtree(destination)
-    destination.mkdir(exist_ok=True)
-    # parse source
+def _parse_package(source: Path):
+    """Parse validators package."""
     v_ast = parse(source.read_text(), source)
-    # generate reference content
     for namespace in (node for node in v_ast.body if isinstance(node, ImportFrom)):
         if not namespace.module:
             continue
-        for alias in namespace.names:
-            ref_module = destination / f"{namespace.module}.md"
-            _write_ref_content(ref_module, namespace.module, alias.name)
-        nav_items["Code Reference"].append(f"reference/{namespace.module}.md")
+        yield (namespace.module, namespace.names)
+
+
+def _generate_reference(source: Path, destination: Path, ext: str):
+    """Generate reference."""
+    nav_items: Dict[str, List[str]] = {"Code Reference": []}
+    # generate reference content
+    for module_name, aliases in _parse_package(source):
+        for alias in aliases:
+            _write_ref_content(destination / f"{module_name}.{ext}", module_name, alias.name)
+        if ext == "md":
+            nav_items["Code Reference"].append(f"reference/{module_name}.md")
     return nav_items
 
 
@@ -54,27 +64,72 @@ def _update_mkdocs_config(source: Path, destination: Path, nav_items: Dict[str,
         safe_dump(mkdocs_conf, mkf, sort_keys=False)
 
 
-def generate_documentation(source: Path, discard_refs: bool = True):
-    """Generate documentation."""
-    # copy readme as docs index file
-    copy(source / "README.md", source / "docs/index.md")
-    # generate reference documentation
-    nav_items = _generate_reference(source / "validators/__init__.py", source / "docs/reference")
+def _gen_md_docs(source: Path, refs_path: Path):
+    """Generate Markdown docs."""
+    nav_items = _generate_reference(source / "validators/__init__.py", refs_path, "md")
     # backup mkdocs config
-    _update_mkdocs_config(source / "mkdocs.yaml", source / "mkdocs.bak.yml", nav_items)
-    # build docs as subprocess
+    _update_mkdocs_config(source / "mkdocs.yaml", source / "mkdocs.bak.yaml", nav_items)
+    # build mkdocs as subprocess
     print(run(("mkdocs", "build"), capture_output=True).stderr.decode())
     # restore mkdocs config
-    move(str(source / "mkdocs.bak.yml"), source / "mkdocs.yaml")
+    move(str(source / "mkdocs.bak.yaml"), source / "mkdocs.yaml")
+
+
+def _gen_rst_docs(source: Path, refs_path: Path):
+    """Generate reStructuredText docs.."""
+    # external
+    from pypandoc import convert_file  # type: ignore
+
+    with open(source / "docs/index.rst", "wt") as idx_f:
+        idx_f.write(
+            convert_file(source_file=source / "docs/index.md", format="md", to="rst")
+            + "\n\n.. toctree::"
+            + "\n   :hidden:"
+            + "\n   :maxdepth: 2"
+            + "\n   :caption: Reference:"
+            + "\n   :glob:\n"
+            + "\n   reference/*\n"
+        )
+    # generate RST reference documentation
+    _generate_reference(source / "validators/__init__.py", refs_path, "rst")
+    # build sphinx web pages as subprocess
+    print(run(("sphinx-build", "docs", "docs/_build/web"), capture_output=True).stderr.decode())
+    # build sphinx man pages as subprocess
+    print(
+        run(
+            ("sphinx-build", "-b", "man", "docs", "docs/_build/man"), capture_output=True
+        ).stderr.decode()
+    )
+
+
+def generate_documentation(
+    source: Path, only_md: bool = False, only_rst: bool = False, discard_refs: bool = True
+):
+    """Generate documentation."""
+    if only_md and only_rst:
+        return
+    # copy readme as docs index file
+    copy(source / "README.md", source / "docs/index.md")
+    # clean destination
+    refs_path = source / "docs/reference"
+    if refs_path.exists() and refs_path.is_dir():
+        rmtree(refs_path)
+    refs_path.mkdir(exist_ok=True)
+    # documentation for each kind
+    if not only_rst:
+        _gen_md_docs(source, refs_path)
+    if not only_md:
+        _gen_rst_docs(source, refs_path)
     # optionally discard reference folder
     if discard_refs:
         rmtree(source / "docs/reference")
 
 
 if __name__ == "__main__":
     project_root = Path(__file__).parent.parent
-    generate_documentation(project_root)
-    # NOTE: use following lines only for testing/debugging
-    # generate_documentation(project_root, discard_refs=False)
-    # from sys import argv
-    # generate_documentation(project_root, len(argv) > 1 and argv[1] == "--keep")
+    generate_documentation(
+        project_root,
+        only_md=True,
+        only_rst=False,
+        discard_refs=len(argv) <= 1 or argv[1] != "--keep",
+    )

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd