feat(replay): Fix truncated JSON bodies

Author: mydea

static/app/utils/replays/replay.tsx

@@ -2,6 +2,7 @@ type JsonObject = Record<string, unknown>;
 type JsonArray = unknown[];
 
 export type NetworkMetaWarning =
+  | 'MAYBE_JSON_TRUNCATED'
   | 'JSON_TRUNCATED'
   | 'TEXT_TRUNCATED'
   | 'INVALID_JSON'

static/app/views/replays/detail/network/details/components.tsx

@@ -20,14 +20,14 @@ const WarningText = styled('span')`
   color: ${p => p.theme.errorText};
 `;
 
-export function Warning({warnings}: {warnings: undefined | string[]}) {
-  if (warnings?.includes('JSON_TRUNCATED') || warnings?.includes('TEXT_TRUNCATED')) {
+export function Warning({warnings}: {warnings: string[]}) {
+  if (warnings.includes('JSON_TRUNCATED') || warnings.includes('TEXT_TRUNCATED')) {
     return (
       <WarningText>{t('Truncated (~~) due to exceeding 150k characters')}</WarningText>
     );
   }
 
-  if (warnings?.includes('INVALID_JSON')) {
+  if (warnings.includes('INVALID_JSON')) {
     return <WarningText>{t('Invalid JSON')}</WarningText>;
   }
 

static/app/views/replays/detail/network/details/sections.tsx

@@ -8,6 +8,10 @@ import {useReplayContext} from 'sentry/components/replays/replayContext';
 import {t} from 'sentry/locale';
 import {space} from 'sentry/styles/space';
 import {formatBytesBase10} from 'sentry/utils';
+import {
+  NetworkMetaWarning,
+  ReplayNetworkRequestOrResponse,
+} from 'sentry/utils/replays/replay';
 import {
   getFrameMethod,
   getFrameStatus,
@@ -24,6 +28,7 @@ import {
   Warning,
 } from 'sentry/views/replays/detail/network/details/components';
 import {useDismissReqRespBodiesAlert} from 'sentry/views/replays/detail/network/details/onboarding';
+import {fixJson} from 'sentry/views/replays/detail/network/truncateJson/fixJson';
 import TimestampButton from 'sentry/views/replays/detail/timestampButton';
 
 export type SectionProps = {
@@ -39,9 +44,6 @@ export function GeneralSection({item, startTimestampMs}: SectionProps) {
 
   const requestFrame = isRequestFrame(item) ? item : null;
 
-  // TODO[replay]: what about:
-  // `requestFrame?.data?.request?.size` vs. `requestFrame?.data?.requestBodySize`
-
   const data: KeyValueTuple[] = [
     {key: t('URL'), value: item.description},
     {key: t('Type'), value: item.op},
@@ -179,6 +181,8 @@ export function RequestPayloadSection({item}: SectionProps) {
   const {dismiss, isDismissed} = useDismissReqRespBodiesAlert();
 
   const data = useMemo(() => (isRequestFrame(item) ? item.data : {}), [item]);
+  const {warnings, body} = getBodyAndWarnings(data.request);
+
   useEffect(() => {
     if (!isDismissed && 'request' in data) {
       dismiss();
@@ -195,9 +199,9 @@ export function RequestPayloadSection({item}: SectionProps) {
       }
     >
       <Indent>
-        <Warning warnings={data.request?._meta?.warnings} />
+        <Warning warnings={warnings} />
         {'request' in data ? (
-          <ObjectInspector data={data.request?.body} expandLevel={2} showCopyButton />
+          <ObjectInspector data={body} expandLevel={2} showCopyButton />
         ) : (
           t('Request body not found.')
         )}
@@ -210,6 +214,8 @@ export function ResponsePayloadSection({item}: SectionProps) {
   const {dismiss, isDismissed} = useDismissReqRespBodiesAlert();
 
   const data = useMemo(() => (isRequestFrame(item) ? item.data : {}), [item]);
+  const {warnings, body} = getBodyAndWarnings(data.response);
+
   useEffect(() => {
     if (!isDismissed && 'response' in data) {
       dismiss();
@@ -226,13 +232,39 @@ export function ResponsePayloadSection({item}: SectionProps) {
       }
     >
       <Indent>
-        <Warning warnings={data?.response?._meta?.warnings} />
+        <Warning warnings={warnings} />
         {'response' in data ? (
-          <ObjectInspector data={data.response?.body} expandLevel={2} showCopyButton />
+          <ObjectInspector data={body} expandLevel={2} showCopyButton />
         ) : (
           t('Response body not found.')
         )}
       </Indent>
     </SectionItem>
   );
 }
+
+function getBodyAndWarnings(reqOrRes?: ReplayNetworkRequestOrResponse): {
+  body: ReplayNetworkRequestOrResponse['body'];
+  warnings: NetworkMetaWarning[];
+} {
+  if (!reqOrRes) {
+    return {body: undefined, warnings: []};
+  }
+
+  const warnings = reqOrRes._meta?.warnings ?? [];
+  let body = reqOrRes.body;
+
+  if (typeof body === 'string' && warnings.includes('MAYBE_JSON_TRUNCATED')) {
+    try {
+      const json = fixJson(body);
+      body = JSON.parse(json);
+      warnings.push('JSON_TRUNCATED');
+    } catch {
+      // this can fail, in which case we just use the body string
+      warnings.push('INVALID_JSON');
+      warnings.push('TEXT_TRUNCATED');
+    }
+  }
+
+  return {body, warnings};
+}

static/app/views/replays/detail/network/truncateJson/completeJson.ts

@@ -0,0 +1,127 @@
+import type {JsonToken} from './constants';
+import {
+  ARR,
+  ARR_VAL,
+  ARR_VAL_COMPLETED,
+  ARR_VAL_STR,
+  OBJ,
+  OBJ_KEY,
+  OBJ_KEY_STR,
+  OBJ_VAL,
+  OBJ_VAL_COMPLETED,
+  OBJ_VAL_STR,
+} from './constants';
+
+const ALLOWED_PRIMITIVES = ['true', 'false', 'null'];
+
+/**
+ * Complete an incomplete JSON string.
+ * This will ensure that the last element always has a `"~~"` to indicate it was truncated.
+ * For example, `[1,2,` will be completed to `[1,2,"~~"]`
+ * and `{"aa":"b` will be completed to `{"aa":"b~~"}`
+ */
+export function completeJson(incompleteJson: string, stack: JsonToken[]): string {
+  if (!stack.length) {
+    return incompleteJson;
+  }
+
+  let json = incompleteJson;
+
+  // Most checks are only needed for the last step in the stack
+  const lastPos = stack.length - 1;
+  const lastStep = stack[lastPos];
+
+  json = _fixLastStep(json, lastStep);
+
+  // Complete remaining steps - just add closing brackets
+  for (let i = lastPos; i >= 0; i--) {
+    const step = stack[i];
+
+    // eslint-disable-next-line default-case
+    switch (step) {
+      case OBJ:
+        json = `${json}}`;
+        break;
+      case ARR:
+        json = `${json}]`;
+        break;
+    }
+  }
+
+  return json;
+}
+
+function _fixLastStep(json: string, lastStep: JsonToken): string {
+  switch (lastStep) {
+    // Object cases
+    case OBJ:
+      return `${json}"~~":"~~"`;
+    case OBJ_KEY:
+      return `${json}:"~~"`;
+    case OBJ_KEY_STR:
+      return `${json}~~":"~~"`;
+    case OBJ_VAL:
+      return _maybeFixIncompleteObjValue(json);
+    case OBJ_VAL_STR:
+      return `${json}~~"`;
+    case OBJ_VAL_COMPLETED:
+      return `${json},"~~":"~~"`;
+
+    // Array cases
+    case ARR:
+      return `${json}"~~"`;
+    case ARR_VAL:
+      return _maybeFixIncompleteArrValue(json);
+    case ARR_VAL_STR:
+      return `${json}~~"`;
+    case ARR_VAL_COMPLETED:
+      return `${json},"~~"`;
+
+    default:
+      return json;
+  }
+}
+
+function _maybeFixIncompleteArrValue(json: string): string {
+  const pos = _findLastArrayDelimiter(json);
+
+  if (pos > -1) {
+    const part = json.slice(pos + 1);
+
+    if (ALLOWED_PRIMITIVES.includes(part.trim())) {
+      return `${json},"~~"`;
+    }
+
+    // Everything else is replaced with `"~~"`
+    return `${json.slice(0, pos + 1)}"~~"`;
+  }
+
+  // fallback, this shouldn't happen, to be save
+  return json;
+}
+
+function _findLastArrayDelimiter(json: string): number {
+  for (let i = json.length - 1; i >= 0; i--) {
+    const char = json[i];
+
+    if (char === ',' || char === '[') {
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+function _maybeFixIncompleteObjValue(json: string): string {
+  const startPos = json.lastIndexOf(':');
+
+  const part = json.slice(startPos + 1);
+
+  if (ALLOWED_PRIMITIVES.includes(part.trim())) {
+    return `${json},"~~":"~~"`;
+  }
+
+  // Everything else is replaced with `"~~"`
+  // This also means we do not have incomplete numbers, e.g `[1` is replaced with `["~~"]`
+  return `${json.slice(0, startPos + 1)}"~~"`;
+}

static/app/views/replays/detail/network/truncateJson/constants.ts

@@ -0,0 +1,23 @@
+export const OBJ = 10;
+export const OBJ_KEY = 11;
+export const OBJ_KEY_STR = 12;
+export const OBJ_VAL = 13;
+export const OBJ_VAL_STR = 14;
+export const OBJ_VAL_COMPLETED = 15;
+
+export const ARR = 20;
+export const ARR_VAL = 21;
+export const ARR_VAL_STR = 22;
+export const ARR_VAL_COMPLETED = 23;
+
+export type JsonToken =
+  | typeof OBJ
+  | typeof OBJ_KEY
+  | typeof OBJ_KEY_STR
+  | typeof OBJ_VAL
+  | typeof OBJ_VAL_STR
+  | typeof OBJ_VAL_COMPLETED
+  | typeof ARR
+  | typeof ARR_VAL
+  | typeof ARR_VAL_STR
+  | typeof ARR_VAL_COMPLETED;