From ae9f625c10d09d3b8cdc8a1421c532e07dd9f98e Mon Sep 17 00:00:00 2001 From: Milan Redele Date: Mon, 19 Aug 2024 10:08:12 +0200 Subject: [PATCH 1/5] fix: always set content-encoding header if content is gzipped --- src/response.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/response.ts b/src/response.ts index 2cd635c..d31e9a6 100644 --- a/src/response.ts +++ b/src/response.ts @@ -63,7 +63,10 @@ export class Response extends EventEmitter { const gzipBase64MagicBytes = 'H4s' let isBase64Gzipped = bodyStr.startsWith(gzipBase64MagicBytes) - if (bodyStr.length > 5000000 && !isBase64Gzipped && this.req.acceptsEncodings('gzip')) { + const needsZipping = + bodyStr.length > 5000000 && !isBase64Gzipped && this.req.acceptsEncodings('gzip') + + if (needsZipping) { // a rough estimate if it won't fit in the 6MB Lambda response limit // with many special characters it might be over the limit bodyStr = gzipSync(bodyStr, { level: 9 }).toString('base64') @@ -72,7 +75,7 @@ export class Response extends EventEmitter { headers['Content-Type'] = 'application/json' } } - if (isBase64Gzipped) { + if (isBase64Gzipped || needsZipping) { headers['Content-Encoding'] = 'gzip' } const apiGatewayResult: APIGatewayProxyResult = { From c2210017028f6f2e038aa2922adcf792dce5e314 Mon Sep 17 00:00:00 2001 From: Milan Redele Date: Mon, 19 Aug 2024 12:04:11 +0200 Subject: [PATCH 2/5] feat: add brotli compression is supported by client --- src/response.spec.js | 135 +++++++++++++++++++++++++++++++++++++++---- src/response.ts | 25 +++++--- 2 files changed, 142 insertions(+), 18 deletions(-) diff --git a/src/response.spec.js b/src/response.spec.js index a730cbe..aa34bc0 100644 --- a/src/response.spec.js +++ b/src/response.spec.js @@ -3,8 +3,85 @@ const { Request } = require('./request') const { gzipSync } = require('zlib') describe('Response object', () => { + const requestObject = { a: 1 } + let req + beforeEach(() => { + const eventV2 = { + version: '2.0', + routeKey: '$default', + rawPath: '/my/path', + rawQueryString: + 'a=1&b=1&b=2&c[]=-firstName&c[]=lastName&d[1]=1&d[0]=0&shoe[color]=yellow&email=test+user@gmail.com&math=1+2&&math=4+5&', + + cookies: ['cookie1', 'cookie2'], + headers: { + 'Content-Type': 'application/json', + 'X-Header': 'value1,value2' + }, + queryStringParameters: { + a: '1', + b: '2', + 'c[]': 'lastName', + 'd[1]': '1', + 'd[0]': '0', + 'shoe[color]': 'yellow', + email: 'test+user@gmail.com', + math: '1+2' + }, + requestContext: { + accountId: '123456789012', + apiId: 'api-id', + authentication: { + clientCert: { + clientCertPem: 'CERT_CONTENT', + subjectDN: 'www.example.com', + issuerDN: 'Example issuer', + serialNumber: 'a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1:a1', + validity: { + notBefore: 'May 28 12:30:02 2019 GMT', + notAfter: 'Aug 5 09:36:04 2021 GMT' + } + } + }, + authorizer: { + jwt: { + claims: { + claim1: 'value1', + claim2: 'value2' + }, + scopes: ['scope1', 'scope2'] + } + }, + domainName: 'id.execute-api.us-east-1.amazonaws.com', + domainPrefix: 'id', + http: { + method: 'POST', + path: '/my/path', + protocol: 'HTTP/1.1', + sourceIp: 'IP', + userAgent: 'agent' + }, + requestId: 'id', + routeKey: '$default', + stage: '$default', + time: '12/Mar/2020:19:03:58 +0000', + timeEpoch: 1583348638390 + }, + body: JSON.stringify(requestObject), + pathParameters: { + parameter1: 'value1' + }, + isBase64Encoded: false, + stageVariables: { + stageVariable1: 'value1', + stageVariable2: 'value2' + } + } + req = new Request(eventV2) + }) + it('set response status properly', done => { - const res = new Response(null, (err, out) => { + const res = new Response(req, (err, out) => { expect(out).toEqual({ statusCode: 404, isBase64Encoded: false, @@ -18,13 +95,48 @@ describe('Response object', () => { }) it('send body properly', done => { - const res = new Response(null, (err, out) => { + const res = new Response(req, (err, out) => { expect(out.body).toBe('hello') done() }) res.send('hello') }) + it('brotli compress large body if supported', done => { + const event = { + headers: { + Accept: 'text/html', + 'Content-Length': 0, + 'Accept-Encoding': 'gzip, deflate, br' + }, + multiValueHeaders: { + Accept: ['text/html'], + 'Content-Length': [0], + 'Accept-Encoding': ['gzip, deflate, br'] + }, + httpMethod: 'POST', + isBase64Encoded: false, + path: '/path', + pathParameters: {}, + queryStringParameters: {}, + multiValueQueryStringParameters: {}, + stageVariables: {}, + requestContext: {}, + resource: '' + } + + const req = new Request(event) + req.next = error => {} + const res = new Response(req, (err, out) => { + expect(out.body).toBeDefined() + expect(out.body.length).toBeLessThan(10000) + expect(out.isBase64Encoded).toBeTruthy() + expect(out.headers['Content-Encoding'] === 'br') + done() + }) + res.send('a'.repeat(6000000)) + }) + it('gzip large body', done => { const event = { headers: { @@ -54,6 +166,7 @@ describe('Response object', () => { expect(out.body).toBeDefined() expect(out.body.length).toBeLessThan(10000) expect(out.isBase64Encoded).toBeTruthy() + expect(out.headers['Content-Encoding'] === 'gzip') done() }) res.send('a'.repeat(6000000)) @@ -95,7 +208,7 @@ describe('Response object', () => { it('already gzipped body left as is', done => { const content = gzipSync('foo bar some text to be zippped...').toString('base64') - const res = new Response(null, (err, out) => { + const res = new Response(req, (err, out) => { expect(out.body).toEqual(content) expect(out.isBase64Encoded).toBeTruthy() done() @@ -104,7 +217,7 @@ describe('Response object', () => { }) it('set content-type', done => { - const res = new Response(null, (err, out) => { + const res = new Response(req, (err, out) => { expect(out.headers).toEqual({ 'content-type': 'text/html' }) @@ -115,7 +228,7 @@ describe('Response object', () => { }) it('get header', done => { - const res = new Response(null, err => { + const res = new Response(req, err => { done() }) res.set('X-Header', 'a') @@ -126,7 +239,7 @@ describe('Response object', () => { }) it('set header with setHeader', done => { - const res = new Response(null, err => { + const res = new Response(req, err => { done() }) res.setHeader('X-Header', 'b') @@ -137,7 +250,7 @@ describe('Response object', () => { }) it('set header with header', done => { - const res = new Response(null, err => { + const res = new Response(req, err => { done() }) res.header('X-Header', 'c') @@ -148,7 +261,7 @@ describe('Response object', () => { }) it('set cookies', done => { - const res = new Response(null, (err, out) => { + const res = new Response(req, (err, out) => { expect(out.multiValueHeaders).toEqual({ 'Set-Cookie': [ 'foo=1234; Path=/', @@ -173,7 +286,7 @@ describe('Response object', () => { }) it('can chain status method', done => { - const res = new Response(null, (err, out) => { + const res = new Response(req, (err, out) => { expect(out.statusCode).toBe(201) expect(res.statusCode).toBe(201) done() @@ -182,7 +295,7 @@ describe('Response object', () => { }) it('can chain set method', done => { - const res = new Response(null, (err, out) => { + const res = new Response(req, (err, out) => { expect(out.headers).toEqual({ 'x-header': 'a' }) done() }) @@ -190,7 +303,7 @@ describe('Response object', () => { }) it('can chain type method', done => { - const response = new Response(null, (err, out) => { + const response = new Response(req, (err, out) => { expect(out.headers).toEqual({ 'content-type': 'text/xml' }) diff --git a/src/response.ts b/src/response.ts index d31e9a6..2c8c4a9 100644 --- a/src/response.ts +++ b/src/response.ts @@ -5,7 +5,7 @@ import { APIGatewayProxyCallbackV2, APIGatewayProxyResult } from 'aws-lambda' -import { gzipSync } from 'zlib' +import { brotliCompressSync, gzipSync } from 'zlib' export class FormatError extends Error { status: number @@ -62,22 +62,33 @@ export class Response extends EventEmitter { const headers = this.expresslessResHeaders const gzipBase64MagicBytes = 'H4s' let isBase64Gzipped = bodyStr.startsWith(gzipBase64MagicBytes) + let isBase64BrotliCompressed = false - const needsZipping = - bodyStr.length > 5000000 && !isBase64Gzipped && this.req.acceptsEncodings('gzip') + const acceptsGzip = this.req.acceptsEncodings('gzip') + const acceptsBrotli = this.req.acceptsEncodings('brotli') + const needsCompression = + bodyStr.length > 5000000 && !isBase64Gzipped && (acceptsGzip || acceptsBrotli) - if (needsZipping) { + if (needsCompression) { // a rough estimate if it won't fit in the 6MB Lambda response limit // with many special characters it might be over the limit - bodyStr = gzipSync(bodyStr, { level: 9 }).toString('base64') - isBase64Gzipped = true + if (acceptsBrotli) { + bodyStr = brotliCompressSync(bodyStr).toString('base64') + isBase64BrotliCompressed = true + } else { + bodyStr = gzipSync(bodyStr, { level: 9 }).toString('base64') + isBase64Gzipped = true + } if (!headers['Content-Type']) { headers['Content-Type'] = 'application/json' } } - if (isBase64Gzipped || needsZipping) { + if (isBase64Gzipped) { headers['Content-Encoding'] = 'gzip' + } else if (isBase64BrotliCompressed) { + headers['Content-Encoding'] = 'br' } + const apiGatewayResult: APIGatewayProxyResult = { statusCode: this.statusCode, headers, From efb259d6606103e3e920ebb4fd5318e57e69af0d Mon Sep 17 00:00:00 2001 From: Milan Redele Date: Mon, 19 Aug 2024 12:19:30 +0200 Subject: [PATCH 3/5] fix: brotli name and params --- src/response.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/response.ts b/src/response.ts index 2c8c4a9..7afa267 100644 --- a/src/response.ts +++ b/src/response.ts @@ -5,7 +5,7 @@ import { APIGatewayProxyCallbackV2, APIGatewayProxyResult } from 'aws-lambda' -import { brotliCompressSync, gzipSync } from 'zlib' +import { brotliCompressSync, gzipSync, constants } from 'zlib' export class FormatError extends Error { status: number @@ -65,7 +65,7 @@ export class Response extends EventEmitter { let isBase64BrotliCompressed = false const acceptsGzip = this.req.acceptsEncodings('gzip') - const acceptsBrotli = this.req.acceptsEncodings('brotli') + const acceptsBrotli = this.req.acceptsEncodings('br') const needsCompression = bodyStr.length > 5000000 && !isBase64Gzipped && (acceptsGzip || acceptsBrotli) @@ -73,7 +73,12 @@ export class Response extends EventEmitter { // a rough estimate if it won't fit in the 6MB Lambda response limit // with many special characters it might be over the limit if (acceptsBrotli) { - bodyStr = brotliCompressSync(bodyStr).toString('base64') + bodyStr = brotliCompressSync(bodyStr, { + params: { + [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT, + [constants.BROTLI_PARAM_SIZE_HINT]: 10000000 + } + }).toString('base64') isBase64BrotliCompressed = true } else { bodyStr = gzipSync(bodyStr, { level: 9 }).toString('base64') @@ -92,7 +97,7 @@ export class Response extends EventEmitter { const apiGatewayResult: APIGatewayProxyResult = { statusCode: this.statusCode, headers, - isBase64Encoded: isBase64Gzipped, + isBase64Encoded: isBase64Gzipped || isBase64BrotliCompressed, body: bodyStr } if (this.expresslessResMultiValueHeaders) From 6e852cd20aecad6889fff332a487fe81b41f2a28 Mon Sep 17 00:00:00 2001 From: Milan Redele Date: Mon, 19 Aug 2024 13:42:56 +0200 Subject: [PATCH 4/5] fix: set lower brotli compression quality to improve speed --- src/response.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/response.ts b/src/response.ts index 7afa267..f363f6b 100644 --- a/src/response.ts +++ b/src/response.ts @@ -76,7 +76,8 @@ export class Response extends EventEmitter { bodyStr = brotliCompressSync(bodyStr, { params: { [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT, - [constants.BROTLI_PARAM_SIZE_HINT]: 10000000 + [constants.BROTLI_PARAM_SIZE_HINT]: 10000000, + [constants.BROTLI_PARAM_QUALITY]: constants.BROTLI_MAX_QUALITY - 2 } }).toString('base64') isBase64BrotliCompressed = true From 4277c9ecf05f16c0882654702aa9d577105c1b30 Mon Sep 17 00:00:00 2001 From: Milan Redele Date: Mon, 19 Aug 2024 13:57:54 +0200 Subject: [PATCH 5/5] fix: set brotli size hint based on input length --- src/response.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/response.ts b/src/response.ts index f363f6b..195d12f 100644 --- a/src/response.ts +++ b/src/response.ts @@ -76,7 +76,7 @@ export class Response extends EventEmitter { bodyStr = brotliCompressSync(bodyStr, { params: { [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT, - [constants.BROTLI_PARAM_SIZE_HINT]: 10000000, + [constants.BROTLI_PARAM_SIZE_HINT]: bodyStr.length, [constants.BROTLI_PARAM_QUALITY]: constants.BROTLI_MAX_QUALITY - 2 } }).toString('base64')