Fix for count being very slow on large Parse Classes' collections (Postgres) (#5330)

* Changed count to be approximate. Should help with postgres slowness

* refactored last commit to only fall back to estimate if no complex query

* handlign variables correctly

* Trying again because it was casting to lowercase table names which doesnt work for us/

* syntax error

* Adding quotations to pg query

* hopefully final pg fix

* Postgres will now use an approximate count unless there is a more complex query specified

* handling edge case

* Fix for count being very slow on large Parse Classes' collections in Postgres. Replicating fix for Mongo in issue 5264

* Fixed silly spelling error resulting from copying over notes

* Lint fixes

* limiting results to 1 on approximation

* suppress test that we can no longer run for postgres

* removed tests from Postgres that no longer apply

* made changes requested by dplewis

* fixed count errors

* updated package.json

* removed test exclude for pg

* removed object types from method

* test disabled for postgres

* returned type

* add estimate count test

* fix mongo test

Author: CoderickLamar

package-lock.json

@@ -145,7 +145,7 @@ describe('AudiencesRouter', () => {
       });
   });
 
-  it('query installations with count = 1', done => {
+  it_exclude_dbs(['postgres'])('query installations with count = 1', done => {
     const config = Config.get('test');
     const androidAudienceRequest = {
       name: 'Android Users',
@@ -189,7 +189,7 @@ describe('AudiencesRouter', () => {
       });
   });
 
-  it('query installations with limit = 0 and count = 1', done => {
+  it_exclude_dbs(['postgres'])('query installations with limit = 0 and count = 1', done => {
     const config = Config.get('test');
     const androidAudienceRequest = {
       name: 'Android Users',

spec/AudienceRouter.spec.js

@@ -160,7 +160,7 @@ describe('InstallationsRouter', () => {
       });
   });
 
-  it('query installations with count = 1', done => {
+  it_exclude_dbs(['postgres'])('query installations with count = 1', done => {
     const config = Config.get('test');
     const androidDeviceRequest = {
       installationId: '12345678-abcd-abcd-abcd-123456789abc',
@@ -209,7 +209,7 @@ describe('InstallationsRouter', () => {
       });
   });
 
-  it('query installations with limit = 0 and count = 1', done => {
+  it_only_db('postgres')('query installations with count = 1', async () => {
     const config = Config.get('test');
     const androidDeviceRequest = {
       installationId: '12345678-abcd-abcd-abcd-123456789abc',
@@ -224,40 +224,90 @@ describe('InstallationsRouter', () => {
       auth: auth.master(config),
       body: {},
       query: {
-        limit: 0,
         count: 1,
       },
       info: {},
     };
 
     const router = new InstallationsRouter();
-    rest
-      .create(
-        config,
-        auth.nobody(config),
-        '_Installation',
-        androidDeviceRequest
-      )
-      .then(() => {
-        return rest.create(
+    await rest.create(
+      config,
+      auth.nobody(config),
+      '_Installation',
+      androidDeviceRequest
+    );
+    await rest.create(
+      config,
+      auth.nobody(config),
+      '_Installation',
+      iosDeviceRequest
+    );
+    let res = await router.handleFind(request);
+    let response = res.response;
+    expect(response.results.length).toEqual(2);
+    expect(response.count).toEqual(0); // estimate count is zero
+
+    const pgAdapter = config.database.adapter;
+    await pgAdapter.updateEstimatedCount('_Installation');
+
+    res = await router.handleFind(request);
+    response = res.response;
+    expect(response.results.length).toEqual(2);
+    expect(response.count).toEqual(2);
+  });
+
+  it_exclude_dbs(['postgres'])(
+    'query installations with limit = 0 and count = 1',
+    done => {
+      const config = Config.get('test');
+      const androidDeviceRequest = {
+        installationId: '12345678-abcd-abcd-abcd-123456789abc',
+        deviceType: 'android',
+      };
+      const iosDeviceRequest = {
+        installationId: '12345678-abcd-abcd-abcd-123456789abd',
+        deviceType: 'ios',
+      };
+      const request = {
+        config: config,
+        auth: auth.master(config),
+        body: {},
+        query: {
+          limit: 0,
+          count: 1,
+        },
+        info: {},
+      };
+
+      const router = new InstallationsRouter();
+      rest
+        .create(
           config,
           auth.nobody(config),
           '_Installation',
-          iosDeviceRequest
-        );
-      })
-      .then(() => {
-        return router.handleFind(request);
-      })
-      .then(res => {
-        const response = res.response;
-        expect(response.results.length).toEqual(0);
-        expect(response.count).toEqual(2);
-        done();
-      })
-      .catch(err => {
-        fail(JSON.stringify(err));
-        done();
-      });
-  });
+          androidDeviceRequest
+        )
+        .then(() => {
+          return rest.create(
+            config,
+            auth.nobody(config),
+            '_Installation',
+            iosDeviceRequest
+          );
+        })
+        .then(() => {
+          return router.handleFind(request);
+        })
+        .then(res => {
+          const response = res.response;
+          expect(response.results.length).toEqual(0);
+          expect(response.count).toEqual(2);
+          done();
+        })
+        .catch(err => {
+          fail(JSON.stringify(err));
+          done();
+        });
+    }
+  );
 });

spec/InstallationsRouter.spec.js

@@ -152,10 +152,7 @@ export class MongoStorageAdapter implements StorageAdapter {
     // encoded
     const encodedUri = formatUrl(parseUrl(this._uri));
 
-    this.connectionPromise = MongoClient.connect(
-      encodedUri,
-      this._mongoOptions
-    )
+    this.connectionPromise = MongoClient.connect(encodedUri, this._mongoOptions)
       .then(client => {
         // Starting mongoDB 3.0, the MongoClient.connect don't return a DB anymore but a client
         // Fortunately, we can get back the options and use them to select the proper DB.
@@ -385,8 +382,8 @@ export class MongoStorageAdapter implements StorageAdapter {
   deleteAllClasses(fast: boolean) {
     return storageAdapterAllCollections(this).then(collections =>
       Promise.all(
-        collections.map(
-          collection => (fast ? collection.deleteMany({}) : collection.drop())
+        collections.map(collection =>
+          fast ? collection.deleteMany({}) : collection.drop()
         )
       )
     );
@@ -952,6 +949,8 @@ export class MongoStorageAdapter implements StorageAdapter {
         readPreference = ReadPreference.NEAREST;
         break;
       case undefined:
+      case null:
+      case '':
         break;
       default:
         throw new Parse.Error(

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -1962,17 +1962,37 @@ export class PostgresStorageAdapter implements StorageAdapter {
   }
 
   // Executes a count.
-  count(className: string, schema: SchemaType, query: QueryType) {
-    debug('count', className, query);
+  count(
+    className: string,
+    schema: SchemaType,
+    query: QueryType,
+    readPreference?: string,
+    estimate?: boolean = true
+  ) {
+    debug('count', className, query, readPreference, estimate);
     const values = [className];
     const where = buildWhereClause({ schema, query, index: 2 });
     values.push(...where.values);
 
     const wherePattern =
       where.pattern.length > 0 ? `WHERE ${where.pattern}` : '';
-    const qs = `SELECT count(*) FROM $1:name ${wherePattern}`;
+    let qs = '';
+
+    if (where.pattern.length > 0 || !estimate) {
+      qs = `SELECT count(*) FROM $1:name ${wherePattern}`;
+    } else {
+      qs =
+        'SELECT reltuples AS approximate_row_count FROM pg_class WHERE relname = $1';
+    }
+
     return this._client
-      .one(qs, values, a => +a.count)
+      .one(qs, values, a => {
+        if (a.approximate_row_count != null) {
+          return +a.approximate_row_count;
+        } else {
+          return +a.count;
+        }
+      })
       .catch(error => {
         if (error.code !== PostgresRelationDoesNotExistError) {
           throw error;
@@ -2327,6 +2347,11 @@ export class PostgresStorageAdapter implements StorageAdapter {
   updateSchemaWithIndexes(): Promise<void> {
     return Promise.resolve();
   }
+
+  // Used for testing purposes
+  updateEstimatedCount(className: string) {
+    return this._client.none('ANALYZE $1:name', [className]);
+  }
 }
 
 function convertPolygonToSQL(polygon) {

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -86,7 +86,8 @@ export interface StorageAdapter {
     className: string,
     schema: SchemaType,
     query: QueryType,
-    readPreference: ?string
+    readPreference?: string,
+    estimate?: boolean
   ): Promise<number>;
   distinct(
     className: string,

src/Adapters/Storage/StorageAdapter.js

@@ -1324,7 +1324,9 @@ class DatabaseController {
       })
       .then((schema: any) => {
         return this.collectionExists(className)
-          .then(() => this.adapter.count(className, { fields: {} }))
+          .then(() =>
+            this.adapter.count(className, { fields: {} }, null, '', false)
+          )
           .then(count => {
             if (count > 0) {
               throw new Parse.Error(