From 6faac31a9096512c053b7249edc6841e9dc871d6 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 27 Jan 2017 05:49:15 +0900 Subject: [PATCH 1/7] =?UTF-8?q?Support=20for=20an=20advanced=20=E2=80=9Cpe?= =?UTF-8?q?rsistentCache=E2=80=9D=20to=20be=20used=20for=20persisting=20ou?= =?UTF-8?q?tside=20of=20the=20current=20memory.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- index.js | 47 ++++++++++++++++++----- readme.markdown | 32 ++++++++++++++++ test/cache_persistent.js | 81 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 test/cache_persistent.js diff --git a/index.js b/index.js index 6684064..90c8994 100644 --- a/index.js +++ b/index.js @@ -27,6 +27,11 @@ function Deps (opts) { if (!opts) opts = {}; this.basedir = opts.basedir || process.cwd(); + this.persistentCache = opts.persistentCache || function (file, id, pkg, fallback, cb) { + setImmediate(function () { + fallback(null, cb); + }); + }; this.cache = opts.cache; this.fileCache = opts.fileCache; this.pkgCache = opts.packageCache || {}; @@ -376,19 +381,41 @@ Deps.prototype.walk = function (id, parent, cb) { var c = self.cache && self.cache[file]; if (c) return fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps)); - self.readFile(file, id, pkg) - .pipe(self.getTransforms(fakePath || file, pkg, { - builtin: builtin, - inNodeModules: parent.inNodeModules - })) - .pipe(concat(function (body) { - fromSource(file, body.toString('utf8'), pkg, fakePath); - })) - ; + self.persistentCache(file, id, pkg, function fallback (stream, cb) { + (stream || self.readFile(file, id, pkg)) + .pipe(self.getTransforms(fakePath || file, pkg, { + builtin: builtin, + inNodeModules: parent.inNodeModules + })) + .pipe(concat(function (body) { + var src = body.toString('utf8'); + var deps = getDeps(file, src); + if (deps) { + cb(null, { + source: src, + package: pkg, + deps: deps.reduce(function (deps, dep) { + deps[dep] = true + return deps + }, {}) + }); + } + })); + }, function (err, c) { + if (err) { + self.emit('error', err); + return; + } + fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps)); + }); }); + function getDeps (file, src) { + return rec.noparse ? [] : self.parseDeps(file, src); + } + function fromSource (file, src, pkg, fakePath) { - var deps = rec.noparse ? [] : self.parseDeps(file, src); + var deps = getDeps(file, src); if (deps) fromDeps(file, src, pkg, fakePath, deps); } diff --git a/readme.markdown b/readme.markdown index 8a1566b..ae642cf 100644 --- a/readme.markdown +++ b/readme.markdown @@ -94,6 +94,38 @@ contents for browser fields, main entries, and transforms * `opts.fileCache` - an object mapping filenames to raw source to avoid reading from disk. +* `opts.persistentCache` - a complex cache handler that allows async and persistent + caching of data. A `persistentCache` needs to follow this interface: + ``` + function peristentCache ( + file, // the path to the file that is loaded + id, // the id that is used to reference this file + pkg, // the package that this file belongs to fallback + cb // callback handler that receives the cache data + ) { + if (hasError()) { + return cb(error) // Pass any error to the callback + } + if (isInCache()) { + return cb(null, { + source: fs.readFileSync(file, 'utf8'), // String of the fully processed file + package: pkg, // The package for housekeeping + deps: { + 'id': // id that is used to reference a required file + 'file' // file path to the required file + } + }) + } + // optional (can be null) stream that provides the data from + // the hard disk, can be provided in case the file data is used + // to evaluate the cache identifier + stream = fs.createReadStream(file) + + // fallback to the default reading + fallback(stream, cb) + } + ``` + * `opts.paths` - array of global paths to search. Defaults to splitting on `':'` in `process.env.NODE_PATH` diff --git a/test/cache_persistent.js b/test/cache_persistent.js new file mode 100644 index 0000000..b90f447 --- /dev/null +++ b/test/cache_persistent.js @@ -0,0 +1,81 @@ +var parser = require('../'); +var test = require('tap').test; +var path = require('path'); +var fs = require('fs'); + +var files = { + foo: path.join(__dirname, '/files/foo.js'), + bar: path.join(__dirname, '/files/bar.js') +}; + +test('uses persistent cache', function (t) { + t.plan(1); + var p = parser({ + persistentCache: function (file, id, pkg, fallback, cb) { + if (file === files.bar) { + return fallback(null, cb) + } + cb(null, { + source: 'file at ' + file + '@' + id, + package: pkg, + deps: { './bar': files.bar } + }) + } + }); + p.end({ id: 'foo', file: files.foo, entry: false }); + + var rows = []; + p.on('data', function (row) { rows.push(row) }); + p.on('end', function () { + t.same(rows.sort(cmp), [ + { + id: files.bar, + file: files.bar, + source: fs.readFileSync(files.bar, 'utf8'), + deps: {} + }, + { + id: 'foo', + file: files.foo, + source: 'file at ' + files.foo + '@' + files.foo, + deps: { './bar': files.bar } + } + ].sort(cmp)); + }); +}); + +test('passes persistent cache error through', function (t) { + t.plan(1); + var p = parser({ + persistentCache: function (file, id, pkg, fallback, cb) { + cb(new Error('foo')) + } + }); + p.end({ id: 'foo', file: files.foo, entry: false }); + p.on('error', function (err) { t.equals(err.message, 'foo') }); +}); + +test('allow passing of a different stream', function (t) { + t.plan(1); + var p = parser({ + persistentCache: function (file, id, pkg, fallback, cb) { + fallback(fs.createReadStream(files.bar), cb) + } + }); + p.end({ id: 'foo', file: files.foo, entry: false }); + + var rows = []; + p.on('data', function (row) { rows.push(row) }); + p.on('end', function () { + t.same(rows.sort(cmp), [ + { + id: 'foo', + file: files.foo, + source: fs.readFileSync(files.bar, 'utf8'), + deps: {} + } + ].sort(cmp)); + }); +}); + +function cmp (a, b) { return a.id < b.id ? -1 : 1 } From ef4532a6eaba7f5e6050654a9ae2cf2fd8683c4a Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 27 Jan 2017 08:50:18 +0900 Subject: [PATCH 2/7] Typo --- readme.markdown | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.markdown b/readme.markdown index ae642cf..1d18d0f 100644 --- a/readme.markdown +++ b/readme.markdown @@ -97,7 +97,7 @@ from disk. * `opts.persistentCache` - a complex cache handler that allows async and persistent caching of data. A `persistentCache` needs to follow this interface: ``` - function peristentCache ( + function persistentCache ( file, // the path to the file that is loaded id, // the id that is used to reference this file pkg, // the package that this file belongs to fallback From 37ca0bb572f9217892225c3728cb039cb4089d5c Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Fri, 27 Jan 2017 08:59:45 +0900 Subject: [PATCH 3/7] - Switched from passing on a stream to passing on a string. - Used database instead of filesystem in the cache example. - Moved the fallback handler to a method one scope out. --- index.js | 38 +++++++++++++++++++++++++------------- readme.markdown | 30 +++++++++++++++++++++--------- test/cache_persistent.js | 2 +- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/index.js b/index.js index 90c8994..99b7ab7 100644 --- a/index.js +++ b/index.js @@ -201,10 +201,7 @@ Deps.prototype.resolve = function (id, parent, cb) { Deps.prototype.readFile = function (file, id, pkg) { var self = this; if (xhas(this.fileCache, file)) { - var tr = through(); - tr.push(this.fileCache[file]); - tr.push(null); - return tr; + return toStream(this.fileCache[file]); } var rs = fs.createReadStream(file, { encoding: 'utf8' @@ -381,8 +378,22 @@ Deps.prototype.walk = function (id, parent, cb) { var c = self.cache && self.cache[file]; if (c) return fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps)); - self.persistentCache(file, id, pkg, function fallback (stream, cb) { - (stream || self.readFile(file, id, pkg)) + self.persistentCache(file, id, pkg, persistentCacheFallback, function (err, c) { + if (err) { + self.emit('error', err); + return; + } + fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps)); + }); + + function persistentCacheFallback (dataAsString, cb) { + var stream + if (dataAsString) { + stream = toStream(dataAsString) + } else { + stream = self.readFile(file, id, pkg) + } + stream .pipe(self.getTransforms(fakePath || file, pkg, { builtin: builtin, inNodeModules: parent.inNodeModules @@ -401,13 +412,7 @@ Deps.prototype.walk = function (id, parent, cb) { }); } })); - }, function (err, c) { - if (err) { - self.emit('error', err); - return; - } - fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps)); - }); + } }); function getDeps (file, src) { @@ -577,6 +582,13 @@ function xhas (obj) { return true; } +function toStream (dataAsString) { + var tr = through(); + tr.push(dataAsString); + tr.push(null); + return tr; +} + function has (obj, key) { return obj && Object.prototype.hasOwnProperty.call(obj, key); } diff --git a/readme.markdown b/readme.markdown index 1d18d0f..c6b3e6f 100644 --- a/readme.markdown +++ b/readme.markdown @@ -106,9 +106,13 @@ from disk. if (hasError()) { return cb(error) // Pass any error to the callback } - if (isInCache()) { + + var fileData = fs.readFileSync(file) + var key = keyFromFile(file, fileData) + + if (db.has(key)) { return cb(null, { - source: fs.readFileSync(file, 'utf8'), // String of the fully processed file + source: db.get(key).toString(), package: pkg, // The package for housekeeping deps: { 'id': // id that is used to reference a required file @@ -116,13 +120,21 @@ from disk. } }) } - // optional (can be null) stream that provides the data from - // the hard disk, can be provided in case the file data is used - // to evaluate the cache identifier - stream = fs.createReadStream(file) - - // fallback to the default reading - fallback(stream, cb) + // + // The fallback will process the file in case the file is not + // in cache. + // + // Note that if your implementation doesn't need the file data + // then you can pass `null` instead of the source and the fallback will + // fetch the data by itself. + // + fallback(fileData, function (error, cacheableEntry) { + if (error) { + return cb(error) + } + db.addToCache(key, cacheableEntry) + cb(null, cacheableEntry) + }) } ``` diff --git a/test/cache_persistent.js b/test/cache_persistent.js index b90f447..30a79f1 100644 --- a/test/cache_persistent.js +++ b/test/cache_persistent.js @@ -59,7 +59,7 @@ test('allow passing of a different stream', function (t) { t.plan(1); var p = parser({ persistentCache: function (file, id, pkg, fallback, cb) { - fallback(fs.createReadStream(files.bar), cb) + fallback(fs.readFileSync(files.bar, 'utf8'), cb) } }); p.end({ id: 'foo', file: files.foo, entry: false }); From 6390913792076161faf860a085298b46b3ff0602 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Wed, 1 Feb 2017 13:09:08 +0900 Subject: [PATCH 4/7] Fixed https://github.com/substack/module-deps/pull/124#discussion_r98822099 --- index.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/index.js b/index.js index 99b7ab7..2706025 100644 --- a/index.js +++ b/index.js @@ -387,12 +387,7 @@ Deps.prototype.walk = function (id, parent, cb) { }); function persistentCacheFallback (dataAsString, cb) { - var stream - if (dataAsString) { - stream = toStream(dataAsString) - } else { - stream = self.readFile(file, id, pkg) - } + var stream = dataAsString ? toStream(dataAsString) : self.readFile(file, id, pkg); stream .pipe(self.getTransforms(fakePath || file, pkg, { builtin: builtin, From c3ac60fb5eb59894e8c433b263a9a60330f4732f Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Wed, 1 Feb 2017 13:09:23 +0900 Subject: [PATCH 5/7] Fixes https://github.com/substack/module-deps/pull/124#discussion_r98822191 --- index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 2706025..c4d5a4b 100644 --- a/index.js +++ b/index.js @@ -401,8 +401,8 @@ Deps.prototype.walk = function (id, parent, cb) { source: src, package: pkg, deps: deps.reduce(function (deps, dep) { - deps[dep] = true - return deps + deps[dep] = true; + return deps; }, {}) }); } From d28fee11f373f22e39e7d5bd98ff6061b94b3f72 Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Wed, 1 Feb 2017 13:11:28 +0900 Subject: [PATCH 6/7] Adds missing fallback document Fixes: https://github.com/substack/module-deps/pull/124#discussion_r98822367 --- readme.markdown | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.markdown b/readme.markdown index c6b3e6f..d3327eb 100644 --- a/readme.markdown +++ b/readme.markdown @@ -101,6 +101,7 @@ from disk. file, // the path to the file that is loaded id, // the id that is used to reference this file pkg, // the package that this file belongs to fallback + fallback, // async fallback handler to be called if the cache doesn't hold the given file cb // callback handler that receives the cache data ) { if (hasError()) { From c940a1c076d7f3ab68d74497d2ea51da1619624e Mon Sep 17 00:00:00 2001 From: Martin Heidegger Date: Wed, 1 Feb 2017 13:12:36 +0900 Subject: [PATCH 7/7] Properly named the test for the string passing Fixes: https://github.com/substack/module-deps/pull/124#discussion_r98822788 --- test/cache_persistent.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cache_persistent.js b/test/cache_persistent.js index 30a79f1..8c18734 100644 --- a/test/cache_persistent.js +++ b/test/cache_persistent.js @@ -55,7 +55,7 @@ test('passes persistent cache error through', function (t) { p.on('error', function (err) { t.equals(err.message, 'foo') }); }); -test('allow passing of a different stream', function (t) { +test('allow passing of the raw source as string', function (t) { t.plan(1); var p = parser({ persistentCache: function (file, id, pkg, fallback, cb) {