+ support for multi-language 3rd-party packages

tested with:
  node ./tools/build.js -t all plaintext
  node ./tools/build.js -t all plaintext shexc solidity sparql ttl

not tested for with test/detect or test/markup tests ('cause I don't
know how).

Author: Eric Prud'hommeaux

test/detect/index.js

@@ -7,7 +7,7 @@ const fs       = require('fs').promises;
 const hljs     = require('../../build');
 const path     = require('path');
 const utility  = require('../utility');
-const { getThirdPartyLanguages } = require('../../tools/lib/external_language')
+const { getThirdPartyPackages } = require('../../tools/lib/external_language')
 
 function testAutoDetection(language, {detectPath}) {
   const languagePath = detectPath || utility.buildPath('detect', language);
@@ -34,7 +34,7 @@ function testAutoDetection(language, {detectPath}) {
 
 describe('hljs.highlightAuto()', () => {
   before( async function() {
-    let thirdPartyLanguages = await getThirdPartyLanguages();
+    let thirdPartyPackages = await getThirdPartyPackages();
 
     let languages = hljs.listLanguages();
     describe(`hljs.highlightAuto()`, function() {
@@ -44,11 +44,14 @@ describe('hljs.highlightAuto()', () => {
       });
     });
 
+    // assumes only one package provides the requested module name
     function detectTestDir(name) {
-      let language = thirdPartyLanguages.find((x) => x.name == name);
-      if (language) {
-        return language.detectTestPath;
-      }
+      thirdPartyPackages.forEach(pkg => {
+        const idx = pkg.names.indexOf(name);
+        if (idx !== -1)
+          return pkg.detectTestPaths[idx]
+      });
+      return null;
     }
   });
 

test/markup/index.js

@@ -7,7 +7,7 @@ const hljs     = require('../../build');
 const path     = require('path');
 const utility  = require('../utility');
 
-const { getThirdPartyLanguages } = require("../../tools/lib/external_language")
+const { getThirdPartyPackages } = require("../../tools/lib/external_language")
 
 function testLanguage(language, {testDir}) {
   describe(language, function() {
@@ -45,8 +45,12 @@ describe('highlight() markup', async () => {
       languages.forEach(testLanguage);
     }
 
-    let thirdPartyLanguages = await getThirdPartyLanguages();
-    return thirdPartyLanguages.forEach((l) => testLanguage(l.name,{testDir: l.markupTestPath}));
+    let thirdPartyPackages = await getThirdPartyPackages();
+    return thirdPartyPackages.forEach(
+      (pkg) => pkg.l.markupTestPaths.forEach(
+        testDir => testLanguage(l.name, {testDir})
+      )
+    );
   })
 
   it("adding dynamic tests...", async function() {} ); // this is required to work

tools/lib/external_language.js

@@ -13,28 +13,28 @@ class LanguagePackage {
   async trySrcLanguages() {
     let dir = path.join(this.dir,"src/languages/*");
     let languages = await glob(dir);
-    if (languages[0]) {
-      this.file = path.join(process.cwd(), languages[0]);
-      this.name = path.basename(this.file,".js");
+    if (languages.length) {
+      this.files = languages.map(fn => path.join(process.cwd(), fn));
+      this.names = this.files.map(fn => path.basename(fn,".js"));
       this._bundle = true;
       this._valid = true;
       return true;
-    }
+    } else { return false; }
   }
 
-  get markupTestPath() {
+  get markupTestPaths() {
     if (this.bundle) {
-      return `${this.dir}/test/markup/${this.name}`;
+      return this.names.map(name => `${this.dir}/test/markup/${name}`);
     } else {
-      return `${this.dir}/test/markup`;
+      return [`${this.dir}/test/markup`];
     }
   }
 
-  get detectTestPath() {
+  get detectTestPaths() {
     if (this.bundle) {
-      return `${this.dir}/test/detect/${this.name}`;
+      return this.names.map(name => `${this.dir}/test/detect/${name}`);
     } else {
-      return `${this.dir}/test/detect`;
+      return [`${this.dir}/test/detect`];
     }
   }
 
@@ -50,8 +50,8 @@ class LanguagePackage {
         if (content.match(MODULE_DEFINER)) {
           this.loader = "definer";
         }
-        this.file = file;
-        this.name = path.basename(file,".js");
+        this.files = [file];
+        this.names = [path.basename(file,".js")];
         this._valid = true;
         return true;
       }
@@ -74,17 +74,17 @@ class LanguagePackage {
   }
 }
 
-async function getThirdPartyLanguages() {
-  let languages = [];
-  let otherLanguages = await glob("./extra/*");
-  for (let packageDir of otherLanguages) {
-    let thirdPartyLanguage = new LanguagePackage(packageDir)
-    let valid = await thirdPartyLanguage.valid();
+async function getThirdPartyPackages() {
+  let packages = [];
+  let otherPackages = await glob("./extra/*");
+  for (let packageDir of otherPackages) {
+    let thirdPartyPackage = new LanguagePackage(packageDir)
+    let valid = await thirdPartyPackage.valid();
     if (valid) {
-      languages.push(thirdPartyLanguage)
+      packages.push(thirdPartyPackage)
     }
   }
-  return languages;
+  return packages;
 }
 
-module.exports = { LanguagePackage, getThirdPartyLanguages}
+module.exports = { LanguagePackage, getThirdPartyPackages}

tools/lib/language.js

@@ -9,7 +9,7 @@ const REQUIRES_REGEX = /\/\*.*?Requires: (.*?)\n/s
 const CATEGORY_REGEX = /\/\*.*?Category: (.*?)\n/s
 const LANGUAGE_REGEX = /\/\*.*?Language: (.*?)\n/s
 const {rollupCode} = require("./bundling.js")
-const { getThirdPartyLanguages } = require("./external_language")
+const { getThirdPartyPackages } = require("./external_language")
 
 class Language {
 
@@ -98,11 +98,13 @@ async function getLanguages() {
   fs.readdirSync("./src/languages/").forEach((file) => {
     languages.push(Language.fromFile(file));
   });
-  let extraLanguages = await getThirdPartyLanguages();
-  for (let ext of extraLanguages) {
-    let l = Language.fromFile(ext.file);
-    l.loader = ext.loader;
-    languages.push(l);
+  let extraPackages = await getThirdPartyPackages();
+  for (let ext of extraPackages) {
+    for (let file of ext.files) {
+      let l = Language.fromFile(file);
+      l.loader = ext.loader;
+      languages.push(l);
+    }
   }
   return languages;
 }