diff --git a/lib/ConcatSource.js b/lib/ConcatSource.js index 06bc248..3c7e7f2 100644 --- a/lib/ConcatSource.js +++ b/lib/ConcatSource.js @@ -61,6 +61,7 @@ class ConcatSource extends Source { if (Buffer.isBuffer(bufferOrString)) { buffers.push(bufferOrString); } else { + // This will not happen buffers.push(Buffer.from(bufferOrString, "utf-8")); } } diff --git a/lib/OriginalSource.js b/lib/OriginalSource.js index fd7e92d..59da29a 100644 --- a/lib/OriginalSource.js +++ b/lib/OriginalSource.js @@ -8,8 +8,7 @@ const { getMap, getSourceAndMap } = require("./helpers/getFromStreamChunks"); const splitIntoLines = require("./helpers/splitIntoLines"); const getGeneratedSourceInfo = require("./helpers/getGeneratedSourceInfo"); const Source = require("./Source"); - -const SPLIT_REGEX = /[^\n;{}]+[;{} \r\t]*\n?|[;{} \r\t]+\n?|\n/g; +const splitIntoPotentialTokens = require("./helpers/splitIntoPotentialTokens"); class OriginalSource extends Source { constructor(value, name) { @@ -61,7 +60,7 @@ class OriginalSource extends Source { const finalSource = !!(options && options.finalSource); if (!options || options.columns !== false) { // With column info we need to read all lines and split them - const matches = this._value.match(SPLIT_REGEX); + const matches = splitIntoPotentialTokens(this._value); let line = 1; let column = 0; if (matches !== null) { diff --git a/lib/helpers/splitIntoPotentialTokens.js b/lib/helpers/splitIntoPotentialTokens.js new file mode 100644 index 0000000..0f0bc42 --- /dev/null +++ b/lib/helpers/splitIntoPotentialTokens.js @@ -0,0 +1,41 @@ +// \n = 10 +// ; = 59 +// { = 123 +// } = 125 +// = 32 +// \r = 13 +// \t = 9 + +const splitIntoPotentialTokens = str => { + const len = str.length; + if (len === 0) return null; + const results = []; + let i = 0; + for (; i < len; ) { + const s = i; + block: { + let cc = str.charCodeAt(i); + while (cc !== 10 && cc !== 59 && cc !== 123 && cc !== 125) { + if (++i >= len) break block; + cc = str.charCodeAt(i); + } + while ( + cc === 59 || + cc === 32 || + cc === 123 || + cc === 125 || + cc === 13 || + cc === 9 + ) { + if (++i >= len) break block; + cc = str.charCodeAt(i); + } + if (cc === 10) { + i++; + } + } + results.push(str.slice(s, i)); + } + return results; +}; +module.exports = splitIntoPotentialTokens;