DOCSP-430: Replace genindex.py with genindex.js, and add lunr (#2)

* Replace genindex.py with genindex.js, and add lunr

Author: i80and

.gitignore

@@ -1 +1,2 @@
 /build
+node_modules/

Makefile

@@ -1,14 +1,16 @@
-PYTHON=python
 GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
 
 .PHONY: build serve help
 
 help: ## Show this help message
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
-build: ## Build the documentation under build/<git branch>
+build: | tools/node_modules ## Build the documentation under build/<git branch>
 	hugo -d build/$(GIT_BRANCH)
-	$(PYTHON) -B tools/genindex.py --out build/$(GIT_BRANCH)/tags.json --config config.toml content/tutorials/
+	$(NODE) tools/genindex.js content/tutorials build/$(GIT_BRANCH)/ --config config.toml
 
 serve: ## Host the documentation on port 1313
 	hugo serve
+
+tools/node_modules: tools/package.json
+	cd tools && npm update

tools/genindex.js

@@ -0,0 +1,203 @@
+#!/usr/bin/env node
+'use strict'
+
+const __doc__ = `
+Usage:
+  genindex.js <source> <outputPrefix> --config=<path>
+`
+
+const assert = require('assert')
+const child_process = require('child_process')
+const fs = require('fs')
+const pathModule = require('path')
+const process = require('process')
+
+const docopt = require('docopt')
+const sax = require('sax')
+const toml = require('toml')
+const lunr = require('lunr')
+
+const PAT_HEADMATTER = /^\+\+\+\n([^]+)\n\+\+\+/
+
+// Recursively step through an object and replace any numbers with a number
+// representable in a short ASCII string.
+function truncateNumbers(r) {
+    if (!(r instanceof Object)) { return }
+    for (let child of Object.keys(r)) {
+        if (typeof r[child] === 'number') {
+            r[child] = Number((r[child]).toString().slice(0, 10))
+        }
+        truncateNumbers(r[child])
+    }
+}
+
+const searchIndex = {
+    idx: lunr(function () {
+        this.ref('id')
+        this.field('title', { boost: 20 })
+        this.field('tags', { boost: 15 })
+        this.field('minorTitles', { boost: 5 })
+        this.field('body')
+    }),
+    docId: 0,
+    slugs: [],
+    toJSON: function() {
+        truncateNumbers(searchIndex.idx.tokenStore.tokens)
+        searchIndex.idx.tokenStore.compress()
+        const json = searchIndex.idx.toJSON()
+        json.slugs = searchIndex.slugs
+        return json
+    }
+}
+
+function* walk(root) {
+    const openList = [root]
+    const closedList = new Set()
+    while (openList.length) {
+        const currentRoot = openList.pop()
+        closedList.add(currentRoot)
+
+        for (const filename of fs.readdirSync(currentRoot)) {
+            const path = currentRoot + '/' + filename
+            const stat = fs.statSync(path)
+            if (stat.isFile()) {
+                yield path
+            } else if (stat.isDirectory() && !closedList.has(path)) {
+                openList.push(path)
+            }
+        }
+    }
+}
+
+function parseXML(path, headmatter, xml) {
+    const spawnOutput = child_process.spawnSync('mmark', ['-xml'], {encoding: 'utf-8', input: xml})
+    if (spawnOutput.status != 0) {
+        throw new Error('Command "mmark" failed')
+    }
+
+    const text = `<root>${spawnOutput.output[1]}</root>`
+    const parser = sax.parser(true, {
+        trim: true,
+        normalize: true
+    })
+
+    const doc = {
+        id: searchIndex.docId,
+        title: headmatter.title,
+        tags: headmatter.tags,
+        minorTitles: [],
+        body: []
+    }
+    searchIndex.docId += 1
+    searchIndex.slugs.push(headmatter.slug)
+
+    let sectionDepth = 0
+    let inName = false
+    let error = false
+
+    parser.onerror = function (error) {
+        console.error('Error parsing ' + path)
+        console.error(error)
+        error = true
+    }
+
+    parser.ontext = function (text) {
+        if (inName) {
+            assert.ok(sectionDepth >= 1)
+            if (sectionDepth === 1) {
+                doc.title += ' ' + text
+            } else {
+                doc.minorTitles.push(text)
+            }
+
+            return
+        }
+
+        doc.body.push(text)
+    }
+
+    parser.onopentag = function (node) {
+        if (node.name === 'section') {
+            sectionDepth += 1
+        } else if (node.name === 'name') {
+            inName = true
+        }
+    }
+
+    parser.onclosetag = function (name) {
+        if (name === 'section') {
+            sectionDepth -= 1
+        } else if (name === 'name') {
+            assert.equal(inName, true)
+            inName = false
+        }
+    }
+
+    parser.write(text).close()
+    if (error) { throw new Error('Parse error') }
+
+    doc.title = doc.title.trim()
+    doc.body = doc.body.join(' ').trim()
+
+    return doc
+}
+
+function processFile(path) {
+    const rawdata = fs.readFileSync(path, { encoding: 'utf-8' })
+    const match = rawdata.match(PAT_HEADMATTER)
+    if (!match) {
+        throw new Error('Couldn\'t find headmatter')
+    }
+
+    const rawHeadmatter = match[1]
+    const headmatter = toml.parse(rawHeadmatter)
+    if (!headmatter.slug) {
+      headmatter.slug = pathModule.parse(path).base
+    }
+
+    const searchDoc = parseXML(path, headmatter, rawdata.slice(match[0].length))
+    searchIndex.idx.add(searchDoc)
+
+    return [headmatter.slug, headmatter.title, headmatter.tags]
+}
+
+function main() {
+    const args = docopt.docopt(__doc__)
+    const data = []
+    const tagManifest = toml.parse(fs.readFileSync(args['--config'])).tags || {}
+    let error = false
+
+    for (const path of walk(args['<source>'])) {
+        let headmatter
+        try {
+            headmatter = processFile(path)
+        } catch(err) {
+            console.error(`Error processing ${path}: ${err}`)
+            error = true
+            continue
+        }
+
+        for (const tag of headmatter[2]) {
+            if (tagManifest[tag] === undefined) {
+                console.error(`Unknown tag "${tag}" in ${path}`)
+                error = true
+            }
+        }
+
+        data.push(headmatter)
+    }
+
+    if (error) {
+        process.exit(1)
+    }
+
+    fs.writeFileSync(args['<outputPrefix>'] + 'tags.json', JSON.stringify({
+        tags: tagManifest,
+        pages: data
+    }))
+
+    const searchIndexJSON = searchIndex.toJSON()
+    fs.writeFileSync(args['<outputPrefix>'] + 'search.json', JSON.stringify(searchIndexJSON))
+}
+
+main()

tools/genindex.py

@@ -0,0 +1,8 @@
+{
+  "dependencies" : {
+    "docopt": "0.6.2",
+    "toml" : "2.3.2",
+    "sax": "1.2.2",
+    "lunr": "i80and/lunr.js#19a85e62ae8103a48ce5a1fad507f758cf6016d9"
+  }
+}