fix: handle big integers in incoming events (#495)

* fix: handle big integers in incoming events

An event may have data that contains a BigInt. The builtin `JSON` parser
for JavaScript does not handle the `BigInt` types. The introduced
`json-bigint` dependency (34k) does.

Fixes: #489

Signed-off-by: Lance Ball <[email protected]>

Author: lance

README.md

@@ -162,6 +162,20 @@ const event3 = new CloudEvent({
 });
 ```
 
+### A Note About Big Integers
+
+When parsing JSON data, if a JSON field value is a number, and that number
+is really big, JavaScript loses precision. For example, the Twitter API exposes
+the Tweet ID. This is a large number that exceeds the integer space of `Number`.
+
+In order to address this situation, you can set the environment variable
+`CE_USE_BIG_INT` to the string value `"true"` to enable the use of the
+[`json-bigint`](https://www.npmjs.com/package/json-bigint) package. This
+package is not used by default due to the resulting slowdown in parse speed
+by a factor of 7x.
+
+See for more information: https://github.com/cloudevents/sdk-javascript/issues/489
+
 ### Example Applications
 
 There are a few trivial example applications in
@@ -205,7 +219,7 @@ There you will find Express.js, TypeScript and Websocket examples.
 | HTTP Batch  | :heavy_check_mark:                                    | :heavy_check_mark:                                    |
 | Kafka Binary  | :heavy_check_mark:                                                   | :heavy_check_mark:                                                   |
 | Kafka Structured  | :heavy_check_mark:                                    | :heavy_check_mark:                                    |
-| Kafka Batch  | :heavy_check_mark:                                    | :heavy_check_mark:  
+| Kafka Batch  | :heavy_check_mark:                                    | :heavy_check_mark:
 | MQTT Binary  | :heavy_check_mark:                                                   | :heavy_check_mark:                                                   |
 | MQTT Structured  | :heavy_check_mark:                                    | :heavy_check_mark:                                    |
 

package-lock.json

@@ -113,6 +113,7 @@
     "ajv": "^8.11.0",
     "ajv-formats": "^2.1.1",
     "process": "^0.11.10",
+    "json-bigint": "^1.0.0",
     "util": "^0.12.4",
     "uuid": "^8.3.2"
   },
@@ -121,6 +122,7 @@
     "@types/chai": "^4.2.11",
     "@types/cucumber": "^6.0.1",
     "@types/got": "^9.6.11",
+    "@types/json-bigint": "^1.0.1",
     "@types/mocha": "^7.0.2",
     "@types/node": "^14.14.10",
     "@types/superagent": "^4.1.10",

package.json

@@ -54,6 +54,7 @@ const CONSTANTS = Object.freeze({
     DATA_SCHEMA: "dataschema",
     DATA_BASE64: "data_base64",
   },
+  USE_BIG_INT_ENV: "CE_USE_BIG_INT"
 } as const);
 
 export default CONSTANTS;

src/constants.ts

@@ -3,9 +3,11 @@
  SPDX-License-Identifier: Apache-2.0
 */
 
+import JSONbig from "json-bigint";
 import CONSTANTS from "./constants";
 import { isString, isDefinedOrThrow, isStringOrObjectOrThrow, ValidationError } from "./event/validation";
 
+const __JSON = JSON;
 export abstract class Parser {
   abstract parse(payload: Record<string, unknown> | string | string[] | undefined): unknown;
 }
@@ -36,6 +38,13 @@ export class JSONParser implements Parser {
 
     isDefinedOrThrow(payload, new ValidationError("null or undefined payload"));
     isStringOrObjectOrThrow(payload, new ValidationError("invalid payload type, allowed are: string or object"));
+
+    if (process.env[CONSTANTS.USE_BIG_INT_ENV] === "true") {
+      JSON = JSONbig(({ useNativeBigInt: true })) as JSON;
+    } else {
+      JSON = __JSON;
+    }
+
     const parseJSON = (v: Record<string, unknown> | string): string => (isString(v) ? JSON.parse(v as string) : v);
     return parseJSON(payload);
   }

src/parsers.ts

@@ -75,6 +75,26 @@ describe("HTTP transport", () => {
     expect(body.extboolean).to.equal(false);
   });
 
+  it("Handles big integers in structured mode", () => {
+    process.env[CONSTANTS.USE_BIG_INT_ENV] = "true";
+    const ce = HTTP.toEvent({
+      headers: { "content-type": "application/cloudevents+json" },
+      body: `{"data": 1524831183200260097}`
+    }) as CloudEvent;
+    expect(ce.data).to.equal(1524831183200260097n);
+    process.env[CONSTANTS.USE_BIG_INT_ENV] = undefined;
+  });
+
+  it("Handles big integers in binary mode", () => {
+    process.env[CONSTANTS.USE_BIG_INT_ENV] = "true";
+    const ce = HTTP.toEvent({
+      headers: { "content-type": "application/json", "ce-id": "1234" },
+      body: `{"data": 1524831183200260097}`
+    }) as CloudEvent<Record<string, never>>;
+    expect((ce.data as Record<string, never>).data).to.equal(1524831183200260097n);
+    process.env[CONSTANTS.USE_BIG_INT_ENV] = undefined;
+  });
+
   it("Handles events with no content-type and no datacontenttype", () => {
     const body = "{Something[Not:valid}JSON";
     const message: Message<undefined> = {

test/integration/message_test.ts

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "ES2016",                       /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
+    "target": "ES2020",                       /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
     "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
     "allowJs": true,                          /* Allow javascript files to be compiled. */
     "checkJs": false,                          /* Report errors in .js files. */