diff --git a/.gitignore b/.gitignore index 1a5bf14..f576c8f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ coverage/ run/ .DS_Store *.swp - +.vscode diff --git a/.travis.yml b/.travis.yml index a11d715..9dd01ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: false + language: node_js node_js: - '10' diff --git a/README.md b/README.md index 1642efe..85a70eb 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ A simple http proxy base on egg httpclient. ```bash -$ npm i @eggjs/http-proxy --save +$ npm i --save @eggjs/http-proxy ``` ```js @@ -37,6 +37,23 @@ exports.httpProxy = { }; ``` +## Configuration + +```js +// {app_root}/config/config.default.js + +/** + * @property {Number} timeout - proxy timeout, ms + * @property {Boolean} withCredentials - whether send cookie when cors + * @property {Boolean} cache - whether cache proxy + * @property {Object} cacheOptions - cache options, see https://www.npmjs.com/package/lru-cache + * @property {Object} ignoreHeaders - ignore request/response headers + */ +exports.httpProxy = {}; +``` + +see [config/config.default.js](config/config.default.js) for more detail. + ## Usage ```js @@ -76,23 +93,35 @@ await ctx.proxyRequest('github.com', { async beforeResponse(proxyResult) { proxyResult.headers.addition = 'true'; - // streaming=false should modify `data`, otherwise use stream to handler proxyResult.res yourself + // use streaming=false, then modify `data`, + // otherwise handler `proxyResult.res` as stream yourself, but don't forgot to adjuest content-length proxyResult.data = proxyResult.data.replace('github.com', 'www.github.com'); return proxyResult; }, }); ``` -## Configuration +### cache ```js -// {app_root}/config/config.default.js exports.httpProxy = { - + cache: false, + cacheOptions: { + // get cache id, if undefined then don't cache the request + // calcId(targetUrl, options) { + // if (options.method === 'GET') return targetUrl; + // }, + // maxAge: 1000 * 60 * 60, + // max: 100, + }, }; ``` -see [config/config.default.js](config/config.default.js) for more detail. +control cache case by case: + +```js +await ctx.proxyRequest('github.com', { cache: true, maxAge: 24 * 60 * 60 * 1000 }); +``` ## Questions & Suggestions diff --git a/app/extend/application.js b/app/extend/application.js index dc65222..abe9134 100644 --- a/app/extend/application.js +++ b/app/extend/application.js @@ -2,7 +2,16 @@ 'use strict'; const HttpProxy = require('../../lib/http_proxy'); +const CacheManager = require('../../lib/cache_manager'); + +const INSTANCE = Symbol('Application#httpProxyCache'); module.exports = { HttpProxy, + get httpProxyCache() { + if (!this[INSTANCE]) { + this[INSTANCE] = new CacheManager(this); + } + return this[INSTANCE]; + }, }; diff --git a/app/extend/context.js b/app/extend/context.js index ecd3fca..e7e259b 100644 --- a/app/extend/context.js +++ b/app/extend/context.js @@ -1,6 +1,6 @@ 'use strict'; -const HTTPPROXY = Symbol('context#httpProxy'); +const HTTPPROXY = Symbol('Context#httpProxy'); module.exports = { /** diff --git a/config/config.default.js b/config/config.default.js index df100d9..c5406b7 100644 --- a/config/config.default.js +++ b/config/config.default.js @@ -6,6 +6,8 @@ * @member Config#httpProxy * @property {Number} timeout - proxy timeout, ms * @property {Boolean} withCredentials - whether send cookie when cors + * @property {Boolean} cache - whether cache proxy + * @property {Object} cacheOptions - cache options, see https://www.npmjs.com/package/lru-cache * @property {Object} ignoreHeaders - ignore request/response headers */ exports.httpProxy = { @@ -13,6 +15,17 @@ exports.httpProxy = { withCredentials: false, + cache: false, + cacheOptions: { + // only cache GET by default + calcId(targetUrl, options) { + if (options.method === 'GET') return targetUrl; + }, + // cache 1 min by default + maxAge: 1000 * 60 * 60, + max: 100, + }, + charsetHeaders: '_input_charset', ignoreHeaders: { diff --git a/lib/cache_manager.js b/lib/cache_manager.js new file mode 100644 index 0000000..9d1a1c8 --- /dev/null +++ b/lib/cache_manager.js @@ -0,0 +1,30 @@ +'use strict'; + +const assert = require('assert'); +const LRU = require('lru-cache'); + +class HttpProxyCacheManager { + constructor(app) { + this.app = app; + this.config = app.config.httpProxy.cacheOptions; + assert(this.config.calcId, 'config.httpProxy.cacheOptions.calcId is required to be a function'); + + this.cache = new LRU(this.config); + } + + calcId(targetUrl, options) { + return this.config.calcId(targetUrl, options); + } + + get(key) { + return this.cache.get(key); + } + + set(key, value, options) { + value.res = undefined; + assert(value.data, 'only cache `data`, please use `streaming: false`'); + return this.cache.set(key, value, options.maxAge); + } +} + +module.exports = HttpProxyCacheManager; diff --git a/lib/http_proxy.js b/lib/http_proxy.js index 49b4838..05c56da 100644 --- a/lib/http_proxy.js +++ b/lib/http_proxy.js @@ -1,6 +1,7 @@ 'use strict'; const { URL } = require('url'); +const Stream = require('stream'); const FormStream = require('formstream'); const ContentType = require('content-type'); const address = require('address'); @@ -41,6 +42,7 @@ class HttpProxy { }; if (options.withCredentials === undefined) options.withCredentials = this.config.withCredentials; + if (options.cache === undefined) options.cache = this.config.cache; let urlObj = new URL(ctx.href); urlObj.host = host; @@ -123,18 +125,43 @@ class HttpProxy { } } - // send request const targetUrl = urlObj.toString(); let proxyResult; - try { - proxyResult = await ctx.curl(targetUrl, options); - this.logger.info(`forward:success, status:${proxyResult.status}, targetUrl:${targetUrl}`); - } catch (err) { - this.logger.warn(`forward:fail, status:${err.status}, targetUrl:${targetUrl}`); - throw err; + + // check cache + let cacheId; + let hitCache; + if (options.cache) { + cacheId = this.app.httpProxyCache.calcId(targetUrl, options); + if (cacheId) { + proxyResult = this.app.httpProxyCache.get(cacheId); + if (proxyResult) { + ctx.set('x-proxy-cache', true); + hitCache = true; + } + } + } + + // send request + if (!hitCache) { + try { + proxyResult = await ctx.curl(targetUrl, options); + + if (options.beforeResponse) proxyResult = await options.beforeResponse(proxyResult); + + // store cache + if (cacheId) { + this.app.httpProxyCache.set(cacheId, proxyResult, options); + } + } catch (err) { + this.logger.warn(`forward:fail, status:${err.status}, targetUrl:${targetUrl}`); + throw err; + } } - if (options.beforeResponse) proxyResult = await options.beforeResponse(proxyResult); + this.logger.info(`forward:success, status:${proxyResult.status}, targetUrl:${targetUrl}, hitCache: ${hitCache}`); + + // send response const { headers, status, data, res } = proxyResult; for (const key of Object.keys(headers)) { @@ -145,20 +172,20 @@ class HttpProxy { ctx.status = status; // avoid egg middleware post-handler to override headers, such as x-frame-options - if (data) { - let body = data; + let body = data || res; + if (body instanceof Stream) { + ctx.respond = false; + ctx.res.flushHeaders(); + body.pipe(ctx.res); + } else { if (!Buffer.isBuffer(body) && typeof body !== 'string') { // body: json body = JSON.stringify(body); - ctx.length = Buffer.byteLength(body); } + ctx.length = Buffer.byteLength(body); ctx.respond = false; ctx.res.flushHeaders(); ctx.res.end(body); - } else { - ctx.respond = false; - ctx.res.flushHeaders(); - res.pipe(ctx.res); } } } diff --git a/package.json b/package.json index 30bddf6..a8f52a8 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "dependencies": { "address": "^1.1.2", "content-type": "^1.0.4", - "formstream": "^1.1.0" + "formstream": "^1.1.0", + "lru-cache": "^5.1.1" }, "devDependencies": { "async-busboy": "^1.0.1", diff --git a/test/cache.test.js b/test/cache.test.js new file mode 100644 index 0000000..b1c9d5e --- /dev/null +++ b/test/cache.test.js @@ -0,0 +1,87 @@ +'use strict'; + +const mock = require('egg-mock'); +const assert = require('assert'); +const mockServer = require('./fixtures/mock_server'); + +describe('test/cache.test.js', () => { + let app; + let cache; + + before(async () => { + app = mock.app({ + baseDir: 'apps/cache', + }); + await app.ready(); + cache = app.httpProxyCache.cache; + }); + + beforeEach(() => mockServer.mock()); + afterEach(() => { + cache.reset(); + mockServer.restore(); + }); + + after(() => app.close()); + afterEach(mock.restore); + + it('should cache', async () => { + let res = await app.httpRequest() + .get('/proxy') + .query('name=tz') + .expect(200); + + assert(res.body.path = '/?name=tz'); + assert(!res.headers['x-proxy-cache']); + assert(cache.has('http://example.com/?name=tz')); + // not save res + assert(!cache.get('http://example.com/?name=tz').res); + + res = await app.httpRequest() + .get('/proxy') + .query('name=tz') + .expect(200); + + assert(res.headers['x-proxy-cache']); + assert(res.body.path = '/?name=tz'); + }); + + it('should not cache POST', async () => { + let res = await app.httpRequest() + .post('/proxy/json') + .set('cookie', 'csrfToken=abc') + .set('x-csrf-token', 'abc') + .send({ a: 'b' }) + .expect(200); + + assert(res.body.requestBody.a === 'b'); + assert(!res.headers['x-proxy-cache']); + assert(cache.length === 0); + + mockServer.restore(); + mockServer.mock(); + + res = await app.httpRequest() + .post('/proxy/json') + .set('cookie', 'csrfToken=abc') + .set('x-csrf-token', 'abc') + .send({ a: 'b' }) + .expect(200); + + assert(res.body.requestBody.a === 'b'); + assert(!res.headers[ 'x-proxy-cache' ]); + assert(cache.length === 0); + }); + + it('should not cache when options.cache = false', async () => { + const res = await app.httpRequest() + .get('/proxy/nocache') + .query('name=tz') + .expect(200); + + assert(res.body.path = '/?name=tz'); + assert(!res.headers['x-proxy-cache']); + + assert(cache.length === 0); + }); +}); diff --git a/test/fixtures/apps/cache/app/controller/proxy.js b/test/fixtures/apps/cache/app/controller/proxy.js new file mode 100644 index 0000000..4b3d4e8 --- /dev/null +++ b/test/fixtures/apps/cache/app/controller/proxy.js @@ -0,0 +1,41 @@ +'use strict'; + +const { Controller } = require('egg'); + +class ProxyController extends Controller { + + async _request(host, opts) { + const { ctx } = this; + if (typeof host !== 'string') { + opts = host; + host = 'example.com'; + } + + await ctx.proxyRequest(host, { + rewrite(urlObj) { + urlObj.port = 80; + urlObj.pathname = urlObj.pathname.replace(/^\/proxy/, ''); + return urlObj; + }, + ...opts, + streaming: false, + }); + } + + async index() { + await this._request(); + } + + async nocache() { + await this._request({ + cache: false, + rewrite(urlObj) { + urlObj.port = 80; + urlObj.pathname = '/'; + return urlObj; + }, + }); + } +} + +module.exports = ProxyController; diff --git a/test/fixtures/apps/cache/app/router.js b/test/fixtures/apps/cache/app/router.js new file mode 100644 index 0000000..5a18ab6 --- /dev/null +++ b/test/fixtures/apps/cache/app/router.js @@ -0,0 +1,10 @@ +'use strict'; + + +module.exports = app => { + const { router, controller } = app; + + router.get('/proxy', controller.proxy.index); + router.post('/proxy/json', controller.proxy.index); + router.get('/proxy/nocache', controller.proxy.nocache); +}; diff --git a/test/fixtures/apps/cache/config/config.default.js b/test/fixtures/apps/cache/config/config.default.js new file mode 100644 index 0000000..5046150 --- /dev/null +++ b/test/fixtures/apps/cache/config/config.default.js @@ -0,0 +1,9 @@ +'use strict'; + +exports.keys = '123456'; + +exports.httpProxy = { + cache: true, + cacheOptions: {}, +}; + diff --git a/test/fixtures/apps/cache/config/plugn.js b/test/fixtures/apps/cache/config/plugn.js new file mode 100644 index 0000000..bcb4215 --- /dev/null +++ b/test/fixtures/apps/cache/config/plugn.js @@ -0,0 +1,3 @@ +'use strict'; + +exports.static = false; diff --git a/test/fixtures/apps/cache/package.json b/test/fixtures/apps/cache/package.json new file mode 100644 index 0000000..8d84309 --- /dev/null +++ b/test/fixtures/apps/cache/package.json @@ -0,0 +1,3 @@ +{ + "name": "cache" +}