From d41df87bd349c680b21133ee1335f695fad89685 Mon Sep 17 00:00:00 2001 From: Eric Prud'hommeaux Date: Thu, 26 Dec 2019 15:04:44 +0100 Subject: [PATCH 1/4] + support for multi-language 3rd-party packages tested with: node ./tools/build.js -t all plaintext node ./tools/build.js -t all plaintext shexc solidity sparql ttl not tested for with test/detect or test/markup tests ('cause I don't know how). --- test/detect/index.js | 15 +++++++----- test/markup/index.js | 10 +++++--- tools/lib/external_language.js | 42 +++++++++++++++++----------------- tools/lib/language.js | 14 +++++++----- 4 files changed, 45 insertions(+), 36 deletions(-) diff --git a/test/detect/index.js b/test/detect/index.js index 13fed6be46..5ec180c167 100644 --- a/test/detect/index.js +++ b/test/detect/index.js @@ -7,7 +7,7 @@ const fs = require('fs').promises; const hljs = require('../../build'); const path = require('path'); const utility = require('../utility'); -const { getThirdPartyLanguages } = require('../../tools/lib/external_language') +const { getThirdPartyPackages } = require('../../tools/lib/external_language') function testAutoDetection(language, {detectPath}) { const languagePath = detectPath || utility.buildPath('detect', language); @@ -34,7 +34,7 @@ function testAutoDetection(language, {detectPath}) { describe('hljs.highlightAuto()', () => { before( async function() { - let thirdPartyLanguages = await getThirdPartyLanguages(); + let thirdPartyPackages = await getThirdPartyPackages(); let languages = hljs.listLanguages(); describe(`hljs.highlightAuto()`, function() { @@ -44,11 +44,14 @@ describe('hljs.highlightAuto()', () => { }); }); + // assumes only one package provides the requested module name function detectTestDir(name) { - let language = thirdPartyLanguages.find((x) => x.name == name); - if (language) { - return language.detectTestPath; - } + thirdPartyPackages.forEach(pkg => { + const idx = pkg.names.indexOf(name); + if (idx !== -1) + return pkg.detectTestPaths[idx] + }); + return null; } }); diff --git a/test/markup/index.js b/test/markup/index.js index 1f5133a065..77d1fd9a4d 100644 --- a/test/markup/index.js +++ b/test/markup/index.js @@ -7,7 +7,7 @@ const hljs = require('../../build'); const path = require('path'); const utility = require('../utility'); -const { getThirdPartyLanguages } = require("../../tools/lib/external_language") +const { getThirdPartyPackages } = require("../../tools/lib/external_language") function testLanguage(language, {testDir}) { describe(language, function() { @@ -45,8 +45,12 @@ describe('highlight() markup', async () => { languages.forEach(testLanguage); } - let thirdPartyLanguages = await getThirdPartyLanguages(); - return thirdPartyLanguages.forEach((l) => testLanguage(l.name,{testDir: l.markupTestPath})); + let thirdPartyPackages = await getThirdPartyPackages(); + return thirdPartyPackages.forEach( + (pkg) => pkg.l.markupTestPaths.forEach( + testDir => testLanguage(l.name, {testDir}) + ) + ); }) it("adding dynamic tests...", async function() {} ); // this is required to work diff --git a/tools/lib/external_language.js b/tools/lib/external_language.js index ec124de7c7..60060141a5 100644 --- a/tools/lib/external_language.js +++ b/tools/lib/external_language.js @@ -13,28 +13,28 @@ class LanguagePackage { async trySrcLanguages() { let dir = path.join(this.dir,"src/languages/*"); let languages = await glob(dir); - if (languages[0]) { - this.file = path.join(process.cwd(), languages[0]); - this.name = path.basename(this.file,".js"); + if (languages.length) { + this.files = languages.map(fn => path.join(process.cwd(), fn)); + this.names = this.files.map(fn => path.basename(fn,".js")); this._bundle = true; this._valid = true; return true; - } + } else { return false; } } - get markupTestPath() { + get markupTestPaths() { if (this.bundle) { - return `${this.dir}/test/markup/${this.name}`; + return this.names.map(name => `${this.dir}/test/markup/${name}`); } else { - return `${this.dir}/test/markup`; + return [`${this.dir}/test/markup`]; } } - get detectTestPath() { + get detectTestPaths() { if (this.bundle) { - return `${this.dir}/test/detect/${this.name}`; + return this.names.map(name => `${this.dir}/test/detect/${name}`); } else { - return `${this.dir}/test/detect`; + return [`${this.dir}/test/detect`]; } } @@ -50,8 +50,8 @@ class LanguagePackage { if (content.match(MODULE_DEFINER)) { this.loader = "definer"; } - this.file = file; - this.name = path.basename(file,".js"); + this.files = [file]; + this.names = [path.basename(file,".js")]; this._valid = true; return true; } @@ -74,17 +74,17 @@ class LanguagePackage { } } -async function getThirdPartyLanguages() { - let languages = []; - let otherLanguages = await glob("./extra/*"); - for (let packageDir of otherLanguages) { - let thirdPartyLanguage = new LanguagePackage(packageDir) - let valid = await thirdPartyLanguage.valid(); +async function getThirdPartyPackages() { + let packages = []; + let otherPackages = await glob("./extra/*"); + for (let packageDir of otherPackages) { + let thirdPartyPackage = new LanguagePackage(packageDir) + let valid = await thirdPartyPackage.valid(); if (valid) { - languages.push(thirdPartyLanguage) + packages.push(thirdPartyPackage) } } - return languages; + return packages; } -module.exports = { LanguagePackage, getThirdPartyLanguages} +module.exports = { LanguagePackage, getThirdPartyPackages} diff --git a/tools/lib/language.js b/tools/lib/language.js index 547128c6f7..1a45ac3b52 100644 --- a/tools/lib/language.js +++ b/tools/lib/language.js @@ -9,7 +9,7 @@ const REQUIRES_REGEX = /\/\*.*?Requires: (.*?)\n/s const CATEGORY_REGEX = /\/\*.*?Category: (.*?)\n/s const LANGUAGE_REGEX = /\/\*.*?Language: (.*?)\n/s const {rollupCode} = require("./bundling.js") -const { getThirdPartyLanguages } = require("./external_language") +const { getThirdPartyPackages } = require("./external_language") class Language { @@ -98,11 +98,13 @@ async function getLanguages() { fs.readdirSync("./src/languages/").forEach((file) => { languages.push(Language.fromFile(file)); }); - let extraLanguages = await getThirdPartyLanguages(); - for (let ext of extraLanguages) { - let l = Language.fromFile(ext.file); - l.loader = ext.loader; - languages.push(l); + let extraPackages = await getThirdPartyPackages(); + for (let ext of extraPackages) { + for (let file of ext.files) { + let l = Language.fromFile(file); + l.loader = ext.loader; + languages.push(l); + } } return languages; } From 5a1df55b463e8c5ff38d07c88fefeec109ecc9ae Mon Sep 17 00:00:00 2001 From: Eric Prud'hommeaux Date: Fri, 27 Dec 2019 19:47:16 +0100 Subject: [PATCH 2/4] ~ fix markup and detect errors https://github.com/highlightjs/highlight.js/pull/2336#issuecomment-569080853 actually two errors: 1 autodetection codepath can return null for non-extra packages 2 'highlight() markup'::before wasn't finished TODO: hljs.highlightAuto() --- test/detect/index.js | 2 +- test/markup/index.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/detect/index.js b/test/detect/index.js index 5ec180c167..578b136664 100644 --- a/test/detect/index.js +++ b/test/detect/index.js @@ -51,7 +51,7 @@ describe('hljs.highlightAuto()', () => { if (idx !== -1) return pkg.detectTestPaths[idx] }); - return null; + return null; // test not found } }); diff --git a/test/markup/index.js b/test/markup/index.js index 77d1fd9a4d..bf3876437c 100644 --- a/test/markup/index.js +++ b/test/markup/index.js @@ -46,9 +46,9 @@ describe('highlight() markup', async () => { } let thirdPartyPackages = await getThirdPartyPackages(); - return thirdPartyPackages.forEach( - (pkg) => pkg.l.markupTestPaths.forEach( - testDir => testLanguage(l.name, {testDir}) + thirdPartyPackages.forEach( + (pkg) => pkg.names.forEach( + (name, idx) => testLanguage(name, {testDir: pkg.markupTestPaths[idx]}) ) ); }) From 115187cae424dd686472b8b7a5f79fc963fa6762 Mon Sep 17 00:00:00 2001 From: Eric Prud'hommeaux Date: Fri, 27 Dec 2019 20:24:40 +0100 Subject: [PATCH 3/4] ~ for the zillionth time, return doesn't work inside a forEach --- test/detect/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/detect/index.js b/test/detect/index.js index 578b136664..30a325f1b3 100644 --- a/test/detect/index.js +++ b/test/detect/index.js @@ -46,11 +46,12 @@ describe('hljs.highlightAuto()', () => { // assumes only one package provides the requested module name function detectTestDir(name) { - thirdPartyPackages.forEach(pkg => { + for (let i = 0; i < thirdPartyPackages.length; ++i) { + const pkg = thirdPartyPackages[i]; const idx = pkg.names.indexOf(name); if (idx !== -1) return pkg.detectTestPaths[idx] - }); + } return null; // test not found } }); From 6704865a0dda9e5c55550046e069d6b5e3af6e37 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Wed, 1 Jan 2020 07:53:46 -0500 Subject: [PATCH 4/4] Be explicit --- tools/lib/external_language.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/external_language.js b/tools/lib/external_language.js index 60060141a5..f9bf49239c 100644 --- a/tools/lib/external_language.js +++ b/tools/lib/external_language.js @@ -13,7 +13,7 @@ class LanguagePackage { async trySrcLanguages() { let dir = path.join(this.dir,"src/languages/*"); let languages = await glob(dir); - if (languages.length) { + if (languages.length > 0) { this.files = languages.map(fn => path.join(process.cwd(), fn)); this.names = this.files.map(fn => path.basename(fn,".js")); this._bundle = true;