- <% _.each(blobs, function(blob) { %>
- <% var categories = blob.fileInfo.Category; %>
-
class="<%= categories.join(' ') %>"<% } %>>
-
<%- blob.fileInfo.Language %>
-
<%- blob.result %>
+ <% _.each(languages, function(language) { %>
+ <% var categories = language.categories; %>
+
0) { %>class="<%= categories.join(' ') %>"<% } %>>
+
<%- language.prettyName %>
+
<%- language.sample %>
<% }); %>
-
+
diff --git a/extra/.keep b/extra/.keep
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/extra/3RD_PARTY_QUICK_START.md b/extra/3RD_PARTY_QUICK_START.md
new file mode 100644
index 0000000000..5a6b73d5f6
--- /dev/null
+++ b/extra/3RD_PARTY_QUICK_START.md
@@ -0,0 +1,70 @@
+*This is a work in progress. PRs to improve these docs (or the process) would be welcome.*
+
+## Getting Started
+
+So you'd like to create and share you're own language for Highlight.js. That's awesome.
+
+Take a look at some of the real-life examples first:
+
+- https://github.com/highlightjs/highlightjs-cypher
+- https://github.com/highlightjs/highlightjs-robots-txt
+
+Basically:
+
+- Checkout highlight-js from github...
+- 3rd party languages are placed into the `extra` directory
+
+So if you had a `xzy` language you'd create an `extra/xyz` folder, and that would be your language module. All paths below are relative to that.
+
+- Put your language file in `src/languages/name.js`.
+- Add detect tests in `test/detect/`.
+- Add markup tests in `test/markup/`.
+- Perhaps add a `package.json` for Node.
+- Add a nice `README`.
+- Don't forget to add a `LICENSE`.
+
+
+## Testing
+
+To test (detect and markup tests), just build highlight.js and test it. Your tests should be automatically run with the suite:
+
+```
+node ./tools/build.js -t node
+npm run test
+```
+
+If you can't get the auto-detect tests passing you should simply turn off auto-detection for your language in it's definition with `disableAutodetect: true`. Auto-detection is hard.
+
+
+## Packaging
+
+Users will expect your package to include a minified CDN distributable in your `dist` folder. This should allow them to add the module to their website with only a single ``).join("");
@@ -41,4 +54,4 @@ const defaultCase = newTestCase({
'"Hello";'
});
-module.exports = { newTestCase, defaultCase, buildFakeDOM };
+module.exports = { newTestCase, defaultCase, buildFakeDOM, findLibrary };
diff --git a/test/browser/worker.js b/test/browser/worker.js
index 3f8a5b11a7..9cc04ef5b2 100644
--- a/test/browser/worker.js
+++ b/test/browser/worker.js
@@ -1,37 +1,30 @@
'use strict';
const Worker = require('tiny-worker');
-const utility = require('../utility');
-const {promisify} = require('util');
-const glob = promisify(require('glob'));
-const {newTestCase, defaultCase } = require('./test_case')
+const {newTestCase, defaultCase, findLibrary } = require('./test_case')
describe('web worker', function() {
- before(function() {
- // Will match both `highlight.pack.js` and `highlight.min.js`
- const filepath = utility.buildPath('..', 'build', 'highlight.*.js');
-
- return glob(filepath).then(hljsPath => {
- this.worker = new Worker(function() {
- self.onmessage = function(event) {
- if (event.data.action === 'importScript') {
- importScripts(event.data.script);
- postMessage(1);
- } else {
- var result = self.hljs.highlight('javascript', event.data);
- postMessage(result.value);
- }
- };
- });
-
- const done = new Promise(resolve => this.worker.onmessage = resolve);
- this.worker.postMessage({
- action: 'importScript',
- script: hljsPath[0]
- });
- return done;
+ before(async function() {
+ this.hljsPath = await findLibrary();
+ this.worker = new Worker(function() {
+ self.onmessage = function(event) {
+ if (event.data.action === 'importScript') {
+ importScripts(event.data.script);
+ postMessage(1);
+ } else {
+ var result = self.hljs.highlight('javascript', event.data);
+ postMessage(result.value);
+ }
+ };
});
+
+ const done = new Promise(resolve => this.worker.onmessage = resolve);
+ this.worker.postMessage({
+ action: 'importScript',
+ script: this.hljsPath
+ });
+ return done;
});
it('should highlight text', function(done) {
diff --git a/test/detect/index.js b/test/detect/index.js
index f22b7ea31f..fec7bb3276 100644
--- a/test/detect/index.js
+++ b/test/detect/index.js
@@ -1,41 +1,62 @@
'use strict';
delete require.cache[require.resolve('../../build')]
-delete require.cache[require.resolve('../../build/lib/highlight')]
+delete require.cache[require.resolve('../../build/lib/core')]
const fs = require('fs').promises;
const hljs = require('../../build');
hljs.debugMode(); // tests run in debug mode so errors are raised
const path = require('path');
const utility = require('../utility');
+const { getThirdPartyPackages } = require('../../tools/lib/external_language')
-function testAutoDetection(language) {
- const languagePath = utility.buildPath('detect', language);
-
- it(`should have test for ${language}`, async () => {
- const path = await fs.stat(languagePath);
- return path.isDirectory().should.be.true;
- });
+function testAutoDetection(language, {detectPath}) {
+ const languagePath = detectPath || utility.buildPath('detect', language);
it(`should be detected as ${language}`, async () => {
- const dirs = await fs.readdir(languagePath)
- const files = await Promise.all(dirs
+ const dir = await fs.stat(languagePath);
+ dir.isDirectory().should.be.true;
+
+ const filenames = await fs.readdir(languagePath)
+ const filesContent = await Promise.all(filenames
.map(function(example) {
const filename = path.join(languagePath, example);
return fs.readFile(filename, 'utf-8');
}))
- files.forEach(function(content) {
- const expected = language,
- actual = hljs.highlightAuto(content).language;
+ filesContent.forEach(function(content) {
+ const expected = language,
+ actual = hljs.highlightAuto(content).language;
- actual.should.equal(expected);
- });
+ actual.should.equal(expected);
+ });
});
}
describe('hljs.highlightAuto()', () => {
- const languages = hljs.listLanguages();
+ before( async function() {
+ let thirdPartyPackages = await getThirdPartyPackages();
+
+ let languages = hljs.listLanguages();
+ describe(`hljs.highlightAuto()`, function() {
+ languages.filter(hljs.autoDetection).forEach((language) => {
+ let detectPath = detectTestDir(language);
+ testAutoDetection(language, { detectPath });
+ });
+ });
+
+ // assumes only one package provides the requested module name
+ function detectTestDir(name) {
+ for (let i = 0; i < thirdPartyPackages.length; ++i) {
+ const pkg = thirdPartyPackages[i];
+ const idx = pkg.names.indexOf(name);
+ if (idx !== -1)
+ return pkg.detectTestPaths[idx]
+ }
+ return null; // test not found
+ }
+ });
- languages.filter(hljs.autoDetection).forEach(testAutoDetection);
+ it("adding dynamic tests...", async function() {} ); // this is required to work
});
+
diff --git a/test/fixtures/nested.js b/test/fixtures/nested.js
index 81e17d3ef2..3c6b59dab9 100644
--- a/test/fixtures/nested.js
+++ b/test/fixtures/nested.js
@@ -12,6 +12,7 @@ module.exports = function(hljs) {
};
BODY.contains = [LIST];
return {
+ disableAutodetect: true,
contains: [LIST]
}
};
diff --git a/test/markup/index.js b/test/markup/index.js
index 01ba8c8a17..bf3876437c 100644
--- a/test/markup/index.js
+++ b/test/markup/index.js
@@ -7,9 +7,14 @@ const hljs = require('../../build');
const path = require('path');
const utility = require('../utility');
-function testLanguage(language) {
+const { getThirdPartyPackages } = require("../../tools/lib/external_language")
+
+function testLanguage(language, {testDir}) {
describe(language, function() {
- const filePath = utility.buildPath('markup', language, '*.expect.txt'),
+ const where = testDir ?
+ path.join(testDir, '*.expect.txt') :
+ utility.buildPath('markup', language, '*.expect.txt');
+ const filePath = where,
filenames = glob.sync(filePath);
_.each(filenames, function(filename) {
@@ -31,13 +36,22 @@ function testLanguage(language) {
});
}
-describe('hljs.highlight()', async () => {
- // TODO: why?
- // ./node_modules/.bin/mocha test/markup
- it("needs this or it can't be run stand-alone", function() {} );
+describe('highlight() markup', async () => {
+ before(async function() {
+ const markupPath = utility.buildPath('markup');
+
+ if (!process.env.ONLY_EXTRA) {
+ let languages = await fs.readdir(markupPath);
+ languages.forEach(testLanguage);
+ }
- const markupPath = utility.buildPath('markup');
+ let thirdPartyPackages = await getThirdPartyPackages();
+ thirdPartyPackages.forEach(
+ (pkg) => pkg.names.forEach(
+ (name, idx) => testLanguage(name, {testDir: pkg.markupTestPaths[idx]})
+ )
+ );
+ })
- const languages = await fs.readdir(markupPath)
- return languages.forEach(testLanguage);
+ it("adding dynamic tests...", async function() {} ); // this is required to work
});
diff --git a/tools/all.js b/tools/all.js
deleted file mode 100644
index 6168be0e53..0000000000
--- a/tools/all.js
+++ /dev/null
@@ -1,34 +0,0 @@
-'use strict';
-
-let _ = require('lodash');
-let path = require('path');
-let cdn = require('./cdn');
-let node = require('./node');
-let browser = require('./browser');
-
-function newBuildDirectory(dir, subdir) {
- const build = path.join(dir.build, subdir);
-
- return { build: build };
-}
-
-module.exports = function(commander, dir) {
- let data = {};
-
- _.each(['cdn', 'node', 'browser'], function(target) {
- const newDirectory = newBuildDirectory(dir, target),
- directory = _.defaults(newDirectory, dir),
- options = _.defaults({ target: target }, commander);
-
- data[target] = {
- directory: directory,
- commander: options
- };
- });
-
- return [].concat(
- cdn(data.cdn.commander, data.cdn.directory),
- node(data.node.commander, data.node.directory),
- browser(data.browser.commander, data.browser.directory)
- );
-};
diff --git a/tools/browser.js b/tools/browser.js
deleted file mode 100644
index a2f6ccdf81..0000000000
--- a/tools/browser.js
+++ /dev/null
@@ -1,143 +0,0 @@
-'use strict';
-
-let _ = require('lodash');
-let bluebird = require('bluebird');
-let readFile = bluebird.promisify(require('fs').readFile);
-let path = require('path');
-
-let registry = require('./tasks');
-let utility = require('./utility');
-
-let directory;
-
-function templateAllFunc(blobs) {
- const name = path.join('demo', 'index.html');
-
- blobs = _.compact(blobs);
-
- return bluebird.join(
- readFile(name),
- utility.getStyleNames(),
- (template, styles) => ({ template, path, blobs, styles })
- );
-}
-
-function copyDocs() {
- const input = path.join(directory.root, 'docs', '*.rst'),
- output = path.join(directory.build, 'docs');
-
- return {
- startLog: { task: ['log', 'Copying documentation.'] },
- read: { requires: 'startLog', task: ['glob', utility.glob(input)] },
- writeLog: { requires: 'read', task: ['log', 'Writing documentation.'] },
- write: { requires: 'writeLog', task: ['dest', output] }
- };
-}
-
-function generateDemo(filterCB, readArgs) {
- let styleDir = path.join('src', 'styles'),
- staticArgs = utility.glob(path.join('demo', '*.min.{js,css}')),
- imageArgs = utility.glob(path.join(styleDir, '*.{png,jpg}'),
- 'binary'),
- stylesArgs = utility.glob(path.join(styleDir, '*.css')),
- demoRoot = path.join(directory.build, 'demo'),
- templateArgs = { callback: templateAllFunc },
- destArgs = {
- dir: path.join(demoRoot, 'styles'),
- encoding: 'binary'
- };
-
- return {
- logStart: { task: ['log', 'Generating demo.'] },
- readLanguages: { requires: 'logStart', task: ['glob', readArgs] },
- filterSnippets: { requires: 'readLanguages', task: ['filter', filterCB] },
- readSnippet: { requires: 'filterSnippets', task: 'readSnippet' },
- template: {
- requires: 'readSnippet',
- task: ['templateAll', templateArgs]
- },
- write: {
- requires: 'template',
- task: ['write', path.join(demoRoot, 'index.html')]
- },
- readStatic: { requires: 'logStart', task: ['glob', staticArgs] },
- writeStatic: { requires: 'readStatic', task: ['dest', demoRoot] },
- readStyles: { requires: 'logStart', task: ['glob', stylesArgs] },
- compressStyles: { requires: 'readStyles', task: 'cssminify' },
- writeStyles: { requires: 'compressStyles', task: ['dest', destArgs] },
- readImages: { requires: 'logStart', task: ['glob', imageArgs] },
- writeImages: { requires:'readImages', task: ['dest', destArgs] },
- readDemoJS: {
- requires: 'logStart',
- task: ['read', path.join('demo', 'demo.js')]
- },
- minifyDemoJS: { requires: 'readDemoJS', task: 'jsminify' },
- writeDemoJS: { requires: 'minifyDemoJS', task: ['dest', demoRoot] },
- readDemoCSS: {
- requires: 'logStart',
- task: ['read', path.join('demo', 'style.css')]
- },
- minifyDemoCSS: { requires: 'readDemoCSS', task: 'cssminify' },
- writeDemoCSS: { requires: 'minifyDemoCSS', task: ['dest', demoRoot] }
- };
-}
-
-module.exports = function(commander, dir) {
- directory = dir;
-
- let hljsExt, output, requiresTask, tasks,
- replace = utility.replace,
- regex = utility.regex,
- replaceClassNames = utility.replaceClassNames,
-
- coreFile = path.join('src', 'highlight.js'),
- languages = utility.glob(path.join('src', 'languages', '*.js')),
- filterCB = utility.buildFilterCallback(commander.args),
- replaceArgs = replace(regex.header, ''),
- templateArgs =
- 'hljs.registerLanguage(\'<%= name %>\', <%= content %>);\n';
-
- tasks = {
- startLog: { task: ['log', 'Building highlight.js pack file.'] },
- readCore: { requires: 'startLog', task: ['read', coreFile] },
- read: { requires: 'startLog', task: ['glob', languages] },
- filter: { requires: 'read', task: ['filter', filterCB] },
- reorder: { requires: 'filter', task: 'reorderDeps' },
- replace: { requires: 'reorder', task: ['replace', replaceArgs] },
- template: { requires: 'replace', task: ['template', templateArgs] },
- packageFiles: {
- requires: ['readCore', 'template'],
- task: 'packageFiles'
- }
- };
- requiresTask = 'packageFiles';
-
- if(commander.compress || commander.target === 'cdn') {
- tasks.minify = { requires: requiresTask, task: 'jsminify' };
- requiresTask = 'minify';
- }
-
- tasks.insertLicenseTag = {
- requires: requiresTask,
- task: 'insertLicenseTag'
- };
-
- tasks.writelog = {
- requires: 'insertLicenseTag',
- task: ['log', 'Writing highlight.js pack file.']
- };
-
- hljsExt = commander.target === 'cdn' ? 'min' : 'pack';
- output = path.join(directory.build, `highlight.${hljsExt}.js`);
-
- tasks.write = {
- requires: 'writelog',
- task: ['write', output]
- };
-
- tasks = (commander.target === 'browser')
- ? [copyDocs(), generateDemo(filterCB, languages), tasks]
- : [tasks];
-
- return utility.toQueue(tasks, registry);
-};
diff --git a/tools/build.js b/tools/build.js
index d39955139b..45ae780309 100644
--- a/tools/build.js
+++ b/tools/build.js
@@ -11,21 +11,20 @@
// * browser
//
// The default target. This will package up the core `highlight.js` along
-// with all the language definitions into the file `highlight.pack.js` --
-// which will be compressed without including the option to disable it. It
-// also builds the documentation for our readthedocs page, mentioned
-// above, along with a local instance of the demo at:
+// with all the language definitions into the file `highlight.js`. A
+// minified version is also created unless `--no-minify` is passed.
+// It also builds the documentation for our readthedocs page, mentioned
+// above, along with a local instance of the demo found at:
//
//
.
//
// * cdn
//
-// This will package up the core `highlight.js` along with all the
-// language definitions into the file `highlight.min.js` and compresses
-// all languages and styles into separate files. Since the target is for
-// CDNs -- like cdnjs and jsdelivr -- it doesn't matter if you put the
-// option to disable compression, this target is always be compressed. Do
-// keep in mind that we don't keep the build results in the main
+// This will package up the core `highlight.js` along with any specified
+// language definitions into the file `highlight.min.js` and also package
+// _all_ languages and styles into separate files. The intended use is for
+// CDNs -- like cdnjs and jsdelivr -- so `--no-minify` is ignored.
+// Do keep in mind that we don't provide the build results in the main
// repository; however, there is a separate repository for those that want
// the CDN builds without using a third party site or building it
// themselves. For those curious, head over to:
@@ -60,30 +59,49 @@
'use strict';
-let commander = require('commander');
-let path = require('path');
-let Queue = require('gear').Queue;
-let registry = require('./tasks');
+const commander = require('commander');
+const path = require('path');
+const { clean } = require("./lib/makestuff")
+const log = (...args) => console.log(...args)
-let build, dir = {};
+const TARGETS = ["cdn", "browser", "node"];
+let dir = {};
commander
.usage('[options] [
...]')
- .option('-n, --no-compress', 'Disable compression')
+ .option('-n, --no-minify', 'Disable minification')
.option('-t, --target ', 'Build for target ' +
'[all, browser, cdn, node]',
- /^(browser|cdn|node|all)$/i, 'browser')
+ 'browser')
.parse(process.argv);
commander.target = commander.target.toLowerCase();
-build = require(`./${commander.target}`);
dir.root = path.dirname(__dirname);
-dir.build = path.join(dir.root, 'build');
+dir.buildRoot = path.join(dir.root, 'build');
-new Queue({ registry: registry })
- .clean(dir.build)
- .log('Starting build.')
- .series(build(commander, dir))
- .log('Finished build.')
- .run();
+async function doTarget(target, buildDir) {
+ const build = require(`./build_${target}`);
+ process.env.BUILD_DIR = buildDir;
+ await clean(buildDir);
+ await build.build({languages: commander.args, minify: commander.minify});
+};
+
+async function doBuild() {
+ log ("Starting build.");
+ if (commander.target=="all") {
+ await clean(dir.buildRoot);
+ for (let target of TARGETS) {
+ log (`** Building ${target.toUpperCase()}. **`);
+ let buildDir = path.join(dir.buildRoot, target);
+ await doTarget(target, buildDir);
+ }
+ } else if (TARGETS.includes(commander.target)) {
+ doTarget(commander.target, dir.buildRoot);
+ } else {
+ log(`ERROR: I do not know how to build '${commander.target}'`);
+ }
+ log ("Finished build.");
+}
+
+doBuild()
diff --git a/tools/build_browser.js b/tools/build_browser.js
new file mode 100644
index 0000000000..6f2a972a33
--- /dev/null
+++ b/tools/build_browser.js
@@ -0,0 +1,150 @@
+const _ = require('lodash');
+const fs = require("fs").promises;
+const glob = require("glob-promise");
+const path = require("path");
+const zlib = require('zlib');
+const Terser = require("terser");
+const child_process = require('child_process');
+const { getLanguages } = require("./lib/language");
+const { filter } = require("./lib/dependencies");
+const config = require("./build_config");
+const { install, install_cleancss, mkdir, renderTemplate } = require("./lib/makestuff");
+const log = (...args) => console.log(...args);
+
+function buildHeader(args) {
+ return "/*\n" +
+ ` Highlight.js ${args.version} (${args.git_sha})\n` +
+ ` License: ${args.license}\n` +
+ ` Copyright (c) ${config.copyrightYears}, ${args.author.name}\n*/`;
+}
+
+async function buildBrowser(options) {
+ var languages = await getLanguages()
+ // filter languages for inclusion in the highlight.js bundle
+ languages = filter(languages, options["languages"]);
+
+ await installDocs();
+ await installDemo(languages);
+
+ log("Preparing languages.")
+ await Promise.all(
+ languages.map(async (lang) => {
+ await lang.compile({terser: config.terser});
+ process.stdout.write(".");
+ } )
+ );
+ log("");
+
+ var size = await buildBrowserHighlightJS(languages, {minify: options.minify})
+
+ log("-----")
+ log("Core :", size.core ,"bytes");
+ if (options.minify)
+ log("Core (min) :", size.core_min ,"bytes");
+ log("Languages :",
+ languages.map((el) => el.data.length).reduce((acc, curr) => acc + curr, 0), "bytes");
+ if (options.minify) {
+ log("Languages (min) :",
+ languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
+ }
+ log("highlight.js :", size.full ,"bytes");
+ if (options.minify) {
+ log("highlight.min.js :", size.minified ,"bytes");
+ log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length ,"bytes");
+ } else {
+ log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length ,"bytes");
+ }
+ log("-----");
+}
+
+async function installDemo(languages) {
+ log("Writing demo files.");
+ mkdir("demo");
+ installDemoStyles();
+
+ const assets = await glob("./demo/*.{js,css}");
+ assets.forEach((file) => install(file));
+
+ const css = await glob("styles/*.css", {cwd:"./src"})
+ const styles = css.map((el) => (
+ { "name": _.startCase(path.basename(el,".css")), "path": el }
+ ));
+ renderTemplate("./demo/index.html", "./demo/index.html", { styles , languages });
+}
+
+async function installDocs() {
+ log("Writing docs files.");
+ mkdir("docs");
+
+ let docs = await glob("./docs/*.rst");
+ docs.forEach((file) => install(file));
+}
+
+function installDemoStyles() {
+ log("Writing style files.");
+ mkdir("demo/styles");
+
+ glob.sync("*", {cwd: "./src/styles"}).forEach((file) => {
+ if (file.endsWith(".css"))
+ install_cleancss(`./src/styles/${file}`,`demo/styles/${file}`);
+ else // images, backgrounds, etc
+ install(`./src/styles/${file}`,`demo/styles/${file}`);
+ })
+}
+
+async function buildBrowserHighlightJS(languages, {minify}) {
+ log("Building highlight.js.");
+
+ var git_sha = child_process
+ .execSync("git rev-parse HEAD")
+ .toString().trim()
+ .slice(0,8)
+ var versionDetails = {...require("../package"), git_sha};
+ var header = buildHeader(versionDetails);
+
+ var outFile = `${process.env.BUILD_DIR}/highlight.js`;
+ var minifiedFile = outFile.replace(/js$/,"min.js");
+ var librarySrc = await fs.readFile("src/highlight.js", {encoding: "utf8"});
+ var coreSize = librarySrc.length;
+
+ // strip off the original top comment
+ librarySrc = librarySrc.replace(/\/\*.*?\*\//s,"");
+
+ var workerStub = "if (typeof importScripts === 'function') { var hljs = self.hljs; }";
+
+ var fullSrc = [
+ header, librarySrc, workerStub,
+ ...languages.map((lang) => lang.module) ].join("\n");
+
+ var tasks = [];
+ tasks.push(fs.writeFile(outFile, fullSrc, {encoding: "utf8"}));
+
+ var core_min = [];
+ var minifiedSrc = "";
+
+ if (minify) {
+ var tersed = Terser.minify(librarySrc, config.terser)
+
+ minifiedSrc = [
+ header, tersed.code, workerStub,
+ ...languages.map((lang) => lang.minified) ].join("\n");
+
+ // get approximate core minified size
+ core_min = [ header, tersed.code, workerStub].join().length;
+
+ tasks.push(fs.writeFile(minifiedFile, minifiedSrc, {encoding: "utf8"}));
+ }
+
+ await Promise.all(tasks);
+ return {
+ core: coreSize,
+ core_min: core_min,
+ minified: Buffer.byteLength(minifiedSrc, 'utf8'),
+ minifiedSrc,
+ fullSrc,
+ full: Buffer.byteLength(fullSrc, 'utf8') };
+}
+
+// CDN build uses the exact same highlight.js distributable
+module.exports.buildBrowserHighlightJS = buildBrowserHighlightJS;
+module.exports.build = buildBrowser;
diff --git a/tools/build_cdn.js b/tools/build_cdn.js
new file mode 100644
index 0000000000..f1de6a0b1d
--- /dev/null
+++ b/tools/build_cdn.js
@@ -0,0 +1,112 @@
+const fs = require("fs").promises;
+const glob = require("glob");
+const zlib = require('zlib');
+const { getLanguages } = require("./lib/language");
+const { filter } = require("./lib/dependencies");
+const config = require("./build_config");
+const { install, install_cleancss, mkdir } = require("./lib/makestuff");
+const log = (...args) => console.log(...args);
+const { buildBrowserHighlightJS } = require("./build_browser");
+const { buildPackageJSON } = require("./build_node");
+const path = require("path");
+
+async function installPackageJSON() {
+ await buildPackageJSON();
+ let json = require(`${process.env.BUILD_DIR}/package`);
+ json.name = "highlight.js-cdn-assets";
+ fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(json, null, ' '));
+}
+
+async function buildCDN(options) {
+ install("./LICENSE", "LICENSE");
+ install("./README.CDN.md","README.md");
+ installPackageJSON();
+
+ installStyles();
+
+ // all the languages are built for the CDN and placed into `/languages`
+ const languages = await getLanguages();
+ await installLanguages(languages);
+
+ // filter languages for inclusion in the highlight.js bundle
+ let embedLanguages = filter(languages, options["languages"])
+
+ // it really makes no sense to embed ALL languages with the CDN build, it's
+ // more likely we want to embed NONE and have completely separate run-time
+ // loading of some sort
+ if (embedLanguages.length == languages.length) {
+ embedLanguages = []
+ }
+
+ var size = await buildBrowserHighlightJS(embedLanguages, {minify: options.minify})
+
+ log("-----")
+ log("Embedded Lang :",
+ embedLanguages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
+ log("All Lang :",
+ languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
+ log("highlight.js :",
+ size.full, "bytes");
+
+ if (options.minify) {
+ log("highlight.min.js :", size.minified ,"bytes");
+ log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length ,"bytes");
+ } else {
+ log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length ,"bytes");
+ }
+ log("-----");
+}
+
+async function installLanguages(languages) {
+ log("Building language files.");
+ mkdir("languages");
+
+ await Promise.all(
+ languages.map(async (language) => {
+ await buildCDNLanguage(language);
+ process.stdout.write(".");
+ })
+ );
+ log("");
+
+ await Promise.all(
+ languages.filter((l) => l.third_party)
+ .map(async (language) => {
+ await buildDistributable(language);
+ })
+ );
+
+ log("");
+}
+
+function installStyles() {
+ log("Writing style files.");
+ mkdir("styles");
+
+ glob.sync("*", {cwd: "./src/styles"}).forEach((file) => {
+ if (file.endsWith(".css"))
+ install_cleancss(`./src/styles/${file}`,`styles/${file.replace(".css",".min.css")}`);
+ else // images, backgrounds, etc
+ install(`./src/styles/${file}`,`styles/${file}`);
+ })
+}
+
+async function buildDistributable(language) {
+ const filename = `${language.name}.min.js`;
+
+ let distDir = path.join(language.moduleDir,"dist")
+ log(`Building ${distDir}/${filename}.`)
+ await fs.mkdir(distDir, {recursive: true});
+ fs.writeFile(path.join(language.moduleDir,"dist",filename), language.minified);
+
+}
+
+ async function buildCDNLanguage (language) {
+ const filename = `${process.env.BUILD_DIR}/languages/${language.name}.min.js`;
+
+ await language.compile({terser: config.terser});
+ fs.writeFile(filename, language.minified);
+}
+
+module.exports.build = buildCDN;
+
diff --git a/tools/build_config.js b/tools/build_config.js
new file mode 100644
index 0000000000..24dcdc8c2b
--- /dev/null
+++ b/tools/build_config.js
@@ -0,0 +1,49 @@
+const cjsPlugin = require('rollup-plugin-commonjs');
+
+module.exports = {
+ build_dir: "build",
+ copyrightYears: "2006-2020",
+ clean_css: {},
+ rollup: {
+ node: {
+ output: { format: "cjs", strict: false },
+ input : {
+ plugins: [
+ cjsPlugin(),
+ {
+ transform: (x) => {
+ if (/var module/.exec(x)) {
+ // remove shim that only breaks things for rollup
+ return x.replace(/var module\s*=.*$/m,"")
+ }
+ }
+ }
+ ],
+ },
+ },
+ browser: {
+ input: {
+ plugins: [
+ cjsPlugin()
+ ]
+ },
+ output: {
+ format: "iife",
+ outro: "return module.exports.definer || module.exports;",
+ strict: false,
+ compact: false,
+ interop: false,
+ extend: false,
+ }
+ }
+ },
+ terser: {
+ "compress": {
+ passes: 2,
+ unsafe: true,
+ warnings: true,
+ dead_code: true,
+ toplevel: "funcs"
+ }
+ }
+}
diff --git a/tools/build_node.js b/tools/build_node.js
new file mode 100644
index 0000000000..14da56d6fc
--- /dev/null
+++ b/tools/build_node.js
@@ -0,0 +1,103 @@
+const fs = require("fs").promises;
+const config = require("./build_config");
+const { getLanguages } = require("./lib/language");
+const { install, mkdir } = require("./lib/makestuff");
+const { filter } = require("./lib/dependencies");
+const { rollupWrite } = require("./lib/bundling.js");
+const log = (...args) => console.log(...args);
+
+async function buildNodeIndex(languages) {
+ const header = "var hljs = require('./core');";
+ const footer = "module.exports = hljs;";
+
+ const registration = languages.map((lang) => {
+ let require = `require('./languages/${lang.name}')`;
+ if (lang.loader) {
+ require = require += `.${lang.loader}`;
+ }
+ return `hljs.registerLanguage('${lang.name}', ${require});`;
+ })
+
+ // legacy
+ await fs.writeFile(`${process.env.BUILD_DIR}/lib/highlight.js`,
+ "// This file has been deprecated in favor of core.js\n" +
+ "var hljs = require('./core');\n"
+ )
+
+ const index = `${header}\n\n${registration.join("\n")}\n\n${footer}`;
+ await fs.writeFile(`${process.env.BUILD_DIR}/lib/index.js`, index);
+}
+
+ async function buildNodeLanguage (language) {
+ const input = { ...config.rollup.node.input, input: language.path }
+ const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/languages/${language.name}.js` }
+ await rollupWrite(input, output)
+}
+
+async function buildNodeHighlightJS() {
+ const input = { input: `src/highlight.js` }
+ const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/core.js` }
+ await rollupWrite(input, output)
+}
+
+async function buildPackageJSON() {
+ const CONTRIBUTOR = /^- (.*) <(.*)>$/
+
+ let authors = await fs.readFile("AUTHORS.txt", {encoding: "utf8"})
+ let lines = authors.split(/\r?\n/)
+ let json = require("../package")
+ json.contributors = lines.reduce((acc, line) => {
+ let matches = line.match(CONTRIBUTOR)
+
+ if (matches) {
+ acc.push({
+ name: matches[1],
+ email: matches[2]
+ })
+ }
+ return acc;
+ }, []);
+ await fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(json, null, ' '));
+}
+
+async function buildLanguages(languages) {
+ log("Writing languages.");
+ await Promise.all(
+ languages.map(async (lang) => {
+ await buildNodeLanguage(lang);
+ process.stdout.write(".");
+ })
+ )
+ log("");
+}
+
+async function buildNode(options) {
+ mkdir("lib/languages");
+ mkdir("scss");
+ mkdir("styles");
+
+ install("./LICENSE", "LICENSE");
+ install("./README.md","README.md");
+
+ log("Writing styles.");
+ const styles = await fs.readdir("./src/styles/");
+ styles.forEach((file) => {
+ install(`./src/styles/${file}`,`styles/${file}`);
+ install(`./src/styles/${file}`,`scss/${file.replace(".css",".scss")}`);
+ })
+ log("Writing package.json.");
+ await buildPackageJSON();
+
+ let languages = await getLanguages()
+ // filter languages for inclusion in the highlight.js bundle
+ languages = filter(languages, options["languages"]);
+
+ await buildNodeIndex(languages);
+ await buildLanguages(languages);
+
+ log("Writing highlight.js");
+ await buildNodeHighlightJS();
+}
+
+module.exports.build = buildNode;
+module.exports.buildPackageJSON = buildPackageJSON;
diff --git a/tools/cdn.js b/tools/cdn.js
deleted file mode 100644
index 3d93c9c11b..0000000000
--- a/tools/cdn.js
+++ /dev/null
@@ -1,88 +0,0 @@
-'use strict';
-
-let path = require('path');
-
-let browserBuild = require('./browser');
-let registry = require('./tasks');
-let utility = require('./utility');
-
-let directory;
-
-function moveLanguages() {
- let input = path.join(directory.root, 'src', 'languages', '*.js'),
- output = path.join(directory.build, 'languages'),
- regex = utility.regex,
- replace = utility.replace,
-
- replaceArgs = replace(regex.header, ''),
- template = 'hljs.registerLanguage(\'<%= name %>\','+
- ' <%= content %>);\n';
-
- return {
- startLog: { task: ['log', 'Building language files.'] },
- read: {
- requires: 'startLog',
- task: ['glob', utility.glob(input)]
- },
- replace: { requires: 'read', task: ['replace', replaceArgs] },
- template: { requires: 'replace', task: ['template', template] },
- replace2: {
- requires: 'template',
- task: [ 'replaceSkippingStrings'
- , replace(regex.replaces, utility.replaceClassNames)
- ]
- },
- replace3: {
- requires: 'replace2',
- task: ['replace', replace(regex.classname, '$1.className')]
- },
- compressLog: {
- requires: 'replace3',
- task: ['log', 'Compressing languages files.']
- },
- minify: { requires: 'compressLog', task: 'jsminify' },
- rename: { requires: 'minify', task: ['rename', { extname: '.min.js' }] },
- writeLog: {
- requires: 'rename',
- task: ['log', 'Writing language files.']
- },
- write: { requires: 'writeLog', task: ['dest', output] }
- };
-}
-
-function moveStyles() {
- const css = path.join(directory.root, 'src', 'styles', '*.css'),
- images = path.join(directory.root, 'src', 'styles', '*.{jpg,png}'),
- output = path.join(directory.build, 'styles'),
- options = { dir: output, encoding: 'binary' };
-
- return {
- startLog: { task: ['log', 'Building style files.'] },
- readCSS: { requires: 'startLog', task: ['glob', utility.glob(css)] },
- readImages: {
- requires: 'startLog',
- task: ['glob', utility.glob(images, 'binary')]
- },
- compressLog: {
- requires: 'readCSS',
- task: ['log', 'Compressing style files.']
- },
- minify: { requires: 'compressLog', task: 'cssminify' },
- rename: {
- requires: 'minify',
- task: ['rename', { extname: '.min.css' }]
- },
- writeLog: {
- requires: ['rename', 'readImages'],
- task: ['log', 'Writing style files.']
- },
- write: { requires: 'writeLog', task: ['dest', options] }
- };
-}
-
-module.exports = function(commander, dir) {
- directory = dir;
-
- return utility.toQueue([moveLanguages(), moveStyles()], registry)
- .concat(browserBuild(commander, dir));
-};
diff --git a/tools/developer.html b/tools/developer.html
index b975a6d8ef..79e334540d 100644
--- a/tools/developer.html
+++ b/tools/developer.html
@@ -68,7 +68,7 @@ highlight.js developer
-
+