feat: Add export all rows of a class and export in JSON format (#2361)

Author: dblythy

src/dashboard/Data/Browser/Browser.react.js

@@ -72,6 +72,8 @@ class Browser extends DashboardView {
       filters: new List(),
       ordering: '-createdAt',
       selection: {},
+      exporting: false,
+      exportingCount: 0,
 
       data: null,
       lastMax: -1,
@@ -1296,15 +1298,12 @@ class Browser extends DashboardView {
     });
   }
 
-  async confirmExportSelectedRows(rows) {
-    this.setState({ rowsToExport: null });
+  async confirmExportSelectedRows(rows, type, indentation) {
+    this.setState({ rowsToExport: null, exporting: true, exportingCount: 0 });
     const className = this.props.params.className;
     const query = new Parse.Query(className);
 
-    if (rows['*']) {
-      // Export all
-      query.limit(10000);
-    } else {
+    if (!rows['*']) {
       // Export selected
       const objectIds = [];
       for (const objectId in this.state.rowsToExport) {
@@ -1314,75 +1313,136 @@ class Browser extends DashboardView {
       query.limit(objectIds.length);
     }
 
-    const classColumns = this.getClassColumns(className, false);
-    // create object with classColumns as property keys needed for ColumnPreferences.getOrder function
-    const columnsObject = {};
-    classColumns.forEach((column) => {
-      columnsObject[column.name] = column;
-    });
-    // get ordered list of class columns
-    const columns = ColumnPreferences.getOrder(
-      columnsObject,
-      this.context.applicationId,
-      className
-    ).filter(column => column.visible);
+    const processObjects = (objects) => {
+      const classColumns = this.getClassColumns(className, false);
+      // create object with classColumns as property keys needed for ColumnPreferences.getOrder function
+      const columnsObject = {};
+      classColumns.forEach((column) => {
+        columnsObject[column.name] = column;
+      });
+      // get ordered list of class columns
+      const columns = ColumnPreferences.getOrder(
+        columnsObject,
+        this.context.applicationId,
+        className
+      ).filter((column) => column.visible);
+
+      if (type === '.json') {
+        const element = document.createElement('a');
+        const file = new Blob(
+          [
+            JSON.stringify(
+              objects.map((obj) => {
+                const json = obj._toFullJSON();
+                delete json.__type;
+                return json;
+              }),
+              null,
+              indentation ? 2 : null,
+            ),
+          ],
+          { type: 'application/json' }
+        );
+        element.href = URL.createObjectURL(file);
+        element.download = `${className}.json`;
+        document.body.appendChild(element); // Required for this to work in FireFox
+        element.click();
+        document.body.removeChild(element);
+        return;
+      }
 
-    const objects = await query.find({ useMasterKey: true });
-    let csvString = columns.map(column => column.name).join(',') + '\n';
-    for (const object of objects) {
-      const row = columns.map(column => {
-        const type = columnsObject[column.name].type;
-        if (column.name === 'objectId') {
-          return object.id;
-        } else if (type === 'Relation' || type === 'Pointer') {
-          if (object.get(column.name)) {
-            return  object.get(column.name).id
-          } else {
-            return ''
-          }
-        } else {
-          let colValue;
-          if (column.name === 'ACL') {
-            colValue = object.getACL();
-          } else {
-            colValue = object.get(column.name);
-          }
-          // Stringify objects and arrays
-          if (Object.prototype.toString.call(colValue) === '[object Object]' || Object.prototype.toString.call(colValue) === '[object Array]') {
-            colValue = JSON.stringify(colValue);
-          }
-          if(typeof colValue === 'string') {
-            if (colValue.includes('"')) {
-              // Has quote in data, escape and quote
-              // If the value contains both a quote and delimiter, adding quotes and escaping will take care of both scenarios
-              colValue = colValue.split('"').join('""');
-              return `"${colValue}"`;
-            } else if (colValue.includes(',')) {
-              // Has delimiter in data, surround with quote (which the value doesn't already contain)
-              return `"${colValue}"`;
+      let csvString = columns.map((column) => column.name).join(',') + '\n';
+      for (const object of objects) {
+        const row = columns
+          .map((column) => {
+            const type = columnsObject[column.name].type;
+            if (column.name === 'objectId') {
+              return object.id;
+            } else if (type === 'Relation' || type === 'Pointer') {
+              if (object.get(column.name)) {
+                return object.get(column.name).id;
+              } else {
+                return '';
+              }
             } else {
-              // No quote or delimiter, just include plainly
-              return `${colValue}`;
+              let colValue;
+              if (column.name === 'ACL') {
+                colValue = object.getACL();
+              } else {
+                colValue = object.get(column.name);
+              }
+              // Stringify objects and arrays
+              if (
+                Object.prototype.toString.call(colValue) ===
+                  '[object Object]' ||
+                Object.prototype.toString.call(colValue) === '[object Array]'
+              ) {
+                colValue = JSON.stringify(colValue);
+              }
+              if (typeof colValue === 'string') {
+                if (colValue.includes('"')) {
+                  // Has quote in data, escape and quote
+                  // If the value contains both a quote and delimiter, adding quotes and escaping will take care of both scenarios
+                  colValue = colValue.split('"').join('""');
+                  return `"${colValue}"`;
+                } else if (colValue.includes(',')) {
+                  // Has delimiter in data, surround with quote (which the value doesn't already contain)
+                  return `"${colValue}"`;
+                } else {
+                  // No quote or delimiter, just include plainly
+                  return `${colValue}`;
+                }
+              } else if (colValue === undefined) {
+                // Export as empty CSV field
+                return '';
+              } else {
+                return `${colValue}`;
+              }
             }
-          } else if (colValue === undefined) {
-            // Export as empty CSV field
-            return '';
-          } else {
-            return `${colValue}`;
+          })
+          .join(',');
+        csvString += row + '\n';
+      }
+
+      // Deliver to browser to download file
+      const element = document.createElement('a');
+      const file = new Blob([csvString], { type: 'text/csv' });
+      element.href = URL.createObjectURL(file);
+      element.download = `${className}.csv`;
+      document.body.appendChild(element); // Required for this to work in FireFox
+      element.click();
+      document.body.removeChild(element);
+    };
+
+    if (!rows['*']) {
+      const objects = await query.find({ useMasterKey: true });
+      processObjects(objects);
+      this.setState({ exporting: false, exportingCount: objects.length });
+    } else {
+      let batch = [];
+      query.eachBatch(
+        (obj) => {
+          batch.push(...obj);
+          if (batch.length % 10 === 0) {
+            this.setState({ exportingCount: batch.length });
           }
-        }
-      }).join(',');
-      csvString += row + '\n';
+          const one_gigabyte = Math.pow(2, 30);
+          const size =
+            new TextEncoder().encode(JSON.stringify(batch)).length /
+            one_gigabyte;
+          if (size.length > 1) {
+            processObjects(batch);
+            batch = [];
+          }
+          if (obj.length !== 100) {
+            processObjects(batch);
+            batch = [];
+            this.setState({ exporting: false, exportingCount: 0 });
+          }
+        },
+        { useMasterKey: true }
+      );
     }
-
-    // Deliver to browser to download file
-    const element = document.createElement('a');
-    const file = new Blob([csvString], { type: 'text/csv' });
-    element.href = URL.createObjectURL(file);
-    element.download = `${className}.csv`;
-    document.body.appendChild(element); // Required for this to work in FireFox
-    element.click();
-    document.body.removeChild(element);
   }
 
   getClassRelationColumns(className) {
@@ -1804,8 +1864,10 @@ class Browser extends DashboardView {
         <ExportSelectedRowsDialog
           className={className}
           selection={this.state.rowsToExport}
+          count={this.state.counts[className]}
+          data={this.state.data}
           onCancel={this.cancelExportSelectedRows}
-          onConfirm={() => this.confirmExportSelectedRows(this.state.rowsToExport)}
+          onConfirm={(type, indentation) => this.confirmExportSelectedRows(this.state.rowsToExport, type, indentation)}
         />
       );
     }
@@ -1822,6 +1884,11 @@ class Browser extends DashboardView {
         <Notification note={this.state.lastNote} isErrorNote={false}/>
       );
     }
+    else if (this.state.exporting) {
+      notification = (
+        <Notification note={`Exporting ${this.state.exportingCount}+ objects...`} isErrorNote={false}/>
+      );
+    }
     return (
       <div>
         <Helmet>

src/dashboard/Data/Browser/ExportSelectedRowsDialog.react.js

@@ -5,36 +5,94 @@
  * This source code is licensed under the license found in the LICENSE file in
  * the root directory of this source tree.
  */
-import Modal     from 'components/Modal/Modal.react';
-import React     from 'react';
+import Modal       from 'components/Modal/Modal.react';
+import React       from 'react';
+import Dropdown    from 'components/Dropdown/Dropdown.react';
+import Field       from 'components/Field/Field.react';
+import Label       from 'components/Label/Label.react';
+import Option      from 'components/Dropdown/Option.react';
+import Toggle      from 'components/Toggle/Toggle.react';
+import TextInput   from 'components/TextInput/TextInput.react';
+import styles      from 'dashboard/Data/Browser/ExportSelectedRowsDialog.scss';
 
 export default class ExportSelectedRowsDialog extends React.Component {
   constructor() {
     super();
 
     this.state = {
-      confirmation: ''
+      confirmation: '',
+      exportType: '.csv',
+      indentation: true,
     };
   }
 
   valid() {
+    if (!this.props.selection['*']) {
+      return true;
+    }
+    if (this.state.confirmation !== 'export all') {
+      return false;
+    }
     return true;
   }
 
+  formatBytes(bytes) {
+    if (!+bytes) return '0 Bytes'
+
+    const k = 1024
+    const decimals = 2
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+
+    const i = Math.floor(Math.log(bytes) / Math.log(k))
+
+    return `${parseFloat((bytes / Math.pow(k, i)).toFixed(decimals))} ${sizes[i]}`
+}
+
+
   render() {
     let selectionLength = Object.keys(this.props.selection).length;
+    const fileSize = new TextEncoder().encode(JSON.stringify(this.props.data, null, this.state.exportType === '.json' && this.state.indentation ? 2 : null)).length / this.props.data.length
     return (
       <Modal
         type={Modal.Types.INFO}
         icon='warn-outline'
         title={this.props.selection['*'] ? 'Export all rows?' : (selectionLength === 1 ? 'Export 1 selected row?' : `Export ${selectionLength} selected rows?`)}
-        subtitle={this.props.selection['*'] ? 'Note: Exporting is limited to the first 10,000 rows.' : ''}
         disabled={!this.valid()}
         confirmText='Export'
         cancelText='Cancel'
         onCancel={this.props.onCancel}
-        onConfirm={this.props.onConfirm}>
-        {}
+        onConfirm={() => this.props.onConfirm(this.state.exportType, this.state.indentation)}>
+        {this.props.selection['*'] && <div className={styles.row} >
+          <Label text="Do you really want to export all rows?" description={<span className={styles.label}>Estimated row count: {this.props.count}<br/>Estimated export size: {this.formatBytes(fileSize * this.props.count)}<br/><br/>⚠️ Exporting all rows may severely impact server or database resources.<br/>Large datasets are exported as multiple files of up to 1 GB each.</span>}/>
+        </div>
+        }
+        <Field
+          label={<Label text='Select export type' />}
+          input={
+            <Dropdown
+              value={this.state.exportType}
+              onChange={(exportType) => this.setState({ exportType })}>
+                <Option value='.csv'>.csv</Option>
+                <Option value='.json'>.json</Option>
+            </Dropdown>
+          } />
+        {this.state.exportType === '.json' && <Field
+          label={<Label text='Indentation' />}
+          input={<Toggle value={this.state.indentation} type={Toggle.Types.YES_NO} onChange={(indentation) => {this.setState({indentation})}} />} />
+          }
+        {this.props.selection['*'] && <Field
+          label={
+            <Label
+              text='Confirm this action'
+              description='Enter "export all" to continue.' />
+          }
+          input={
+            <TextInput
+              placeholder='export all'
+              value={this.state.confirmation}
+              onChange={(confirmation) => this.setState({ confirmation })} />
+          } />
+        }
       </Modal>
     );
   }

src/dashboard/Data/Browser/ExportSelectedRowsDialog.scss

@@ -0,0 +1,9 @@
+.row {
+  display: block;
+  position: relative;
+  height: 100px;
+  border-bottom: 1px solid #e0e0e1;
+}
+.label {
+  line-height: 16px;
+}