From c7ffba0dff2c7024a89de4285f95056aa32f6e76 Mon Sep 17 00:00:00 2001 From: Corey Date: Tue, 10 Mar 2020 22:11:40 -0400 Subject: [PATCH 01/50] Update .travis.yml testing error to see what happens... --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f5383e20e0..55012aa0a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ addons: postgresql: '11' apt: packages: - - postgresql-11 - postgresql-11-postgis-2.5 - postgresql-11-postgis-2.5-scripts branches: From 2e5bb8fc730d8cfa30eb794be2f33a2c83133440 Mon Sep 17 00:00:00 2001 From: Corey Date: Tue, 10 Mar 2020 22:49:19 -0400 Subject: [PATCH 02/50] Update .travis.yml Attempting to resolve postgres in CL by installing postgis via sudo instead of through apt/packages --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 55012aa0a2..69e91ed3be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,6 +41,8 @@ before_install: - sudo service postgresql start 11 before_script: - node -e 'require("./lib/index.js")' +- sudo apt-get -qq update +- sudo apt-get install -y postgresql-11-postgis-2.5 postgresql-11-postgis-2.5-scripts - psql -c 'create database parse_server_postgres_adapter_test_database;' -U postgres - psql -c 'CREATE EXTENSION postgis;' -U postgres -d parse_server_postgres_adapter_test_database - psql -c 'CREATE EXTENSION postgis_topology;' -U postgres -d parse_server_postgres_adapter_test_database From 9e1199167c88b6ea195e747dea3a646d619293dd Mon Sep 17 00:00:00 2001 From: Corey Date: Tue, 10 Mar 2020 23:26:56 -0400 Subject: [PATCH 03/50] Update .travis.yml --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 69e91ed3be..91aa7ff5e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,10 @@ services: - docker addons: postgresql: '11' - apt: - packages: - - postgresql-11-postgis-2.5 - - postgresql-11-postgis-2.5-scripts + #apt: + # packages: + # - postgresql-11-postgis-2.5 + # - postgresql-11-postgis-2.5-scripts branches: only: - master From efe493d241cbe6fb6393c19eb665d0c4c2d885fc Mon Sep 17 00:00:00 2001 From: Corey Date: Tue, 10 Mar 2020 23:37:35 -0400 Subject: [PATCH 04/50] Update .travis.yml --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 91aa7ff5e2..d3cefbaeac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,15 @@ language: node_js os: linux dist: xenial services: -- postgresql +#- postgresql - redis - docker addons: postgresql: '11' - #apt: - # packages: - # - postgresql-11-postgis-2.5 - # - postgresql-11-postgis-2.5-scripts + apt: + packages: + - postgresql-11-postgis-2.5 + - postgresql-11-postgis-2.5-scripts branches: only: - master @@ -35,14 +35,14 @@ before_install: - nvm install $NODE_VERSION - nvm use $NODE_VERSION - npm install -g greenkeeper-lockfile@1 +#- sudo apt-get -qq update +#- sudo apt-get install -y postgresql-11-postgis-2.5 postgresql-11-postgis-2.5-scripts - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf - sudo service postgresql stop - sudo service postgresql start 11 before_script: - node -e 'require("./lib/index.js")' -- sudo apt-get -qq update -- sudo apt-get install -y postgresql-11-postgis-2.5 postgresql-11-postgis-2.5-scripts - psql -c 'create database parse_server_postgres_adapter_test_database;' -U postgres - psql -c 'CREATE EXTENSION postgis;' -U postgres -d parse_server_postgres_adapter_test_database - psql -c 'CREATE EXTENSION postgis_topology;' -U postgres -d parse_server_postgres_adapter_test_database From 0dc0cf5ef896ee18072a0878c5492b4a0ebd8a4a Mon Sep 17 00:00:00 2001 From: Corey Date: Tue, 10 Mar 2020 23:54:52 -0400 Subject: [PATCH 05/50] Update .travis.yml Removed extra lines of postgres that were under "services" and "addons". I believe the "postgresql" line under "services" was installing the default of 9.6 and "addons" was installing postgres 11. My guess is the fail was occurring due to 9.6 being called sometimes and it never had postgis installed. If this is true, the solution is to only install one version of postgres, which is version 11 with postgis 2.5. --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d3cefbaeac..c47d33628c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ language: node_js os: linux dist: xenial services: -#- postgresql - redis - docker addons: @@ -35,8 +34,6 @@ before_install: - nvm install $NODE_VERSION - nvm use $NODE_VERSION - npm install -g greenkeeper-lockfile@1 -#- sudo apt-get -qq update -#- sudo apt-get install -y postgresql-11-postgis-2.5 postgresql-11-postgis-2.5-scripts - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/11/main/postgresql.conf - sudo cp /etc/postgresql/{10,11}/main/pg_hba.conf - sudo service postgresql stop From 4af02f02fbc1bdc30ac08cf1fdd0097aa73981a9 Mon Sep 17 00:00:00 2001 From: Corey Date: Thu, 12 Mar 2020 20:56:02 -0400 Subject: [PATCH 06/50] Adding test case for caseInsensitive Adding test case for verifying indexing for caseInsensitive --- spec/PostgresStorageAdapter.spec.js | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index e722fa4cec..9658f4f572 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -16,6 +16,13 @@ const dropTable = (client, className) => { return client.none('DROP TABLE IF EXISTS $', { className }); }; +const getQueryPlan = (client, query) => { + return client.any( + 'EXPLAIN (ANALYZE, FORMAT JSON) $', + { query } + ); +}; + describe_only_db('postgres')('PostgresStorageAdapter', () => { let adapter; beforeEach(() => { @@ -144,6 +151,34 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { undefined ); }); + + it('should use index for caseInsensitive query', async () => { + const user = new Parse.User(); + user.set('username', 'Bugs'); + user.set('password', 'Bunny'); + await user.signUp(); + + const database = Config.get(Parse.applicationId).database; + + const client = adapter._client; + + const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower('bugs')') + + const schema = await new Parse.Schema('_User').get(); + + await database.adapter.ensureIndex( + '_User', + schema, + ['username'], + 'case_insensitive_username', + true + ); + + const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower('bugs')') + + expect(preIndexPlan.executionStats.executionStages.stage).toBe('COLLSCAN'); + expect(postIndexPlan.executionStats.executionStages.stage).toBe('FETCH'); + }); }); describe_only_db('postgres')('PostgresStorageAdapter shutdown', () => { From e3642421be157626e81bbcf3bc8352a20417cbde Mon Sep 17 00:00:00 2001 From: Corey Date: Thu, 12 Mar 2020 20:57:06 -0400 Subject: [PATCH 07/50] Implementing ensureIndex --- .../Storage/Postgres/PostgresStorageAdapter.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 31602afc0d..052f0e12d9 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2524,8 +2524,22 @@ export class PostgresStorageAdapter implements StorageAdapter { } // TODO: implement? - ensureIndex(): Promise { - return Promise.resolve(); + ensureIndex( + className: string, + schema: SchemaType, + fieldNames: string[], + indexName: ?string, + caseInsensitive: boolean = false + ): Promise { + + return this.setIndexesWithSchemaFormat( + className, + schema.indexes, + {}, + schema.fields, + null + ); + //return Promise.resolve(); } } From 8c27d4180997fdd25f10b745528d73683c6010bb Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 12:18:31 -0400 Subject: [PATCH 08/50] Updated PostgresStorageAdapter calls to ST_DistanceSphere. Note this has a minimum requirement of postgis 2.2. Documented the change in the readme. This is address #6441 --- README.md | 2 +- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b935b83e03..b5e5d43d26 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ The fastest and easiest way to get started is to run MongoDB and Parse Server lo Before you start make sure you have installed: - [NodeJS](https://www.npmjs.com/) that includes `npm` -- [MongoDB](https://www.mongodb.com/) or [PostgreSQL](https://www.postgresql.org/) +- [MongoDB](https://www.mongodb.com/) or [PostgreSQL](https://www.postgresql.org/)(with [PostGIS](https://postgis.net) 2.2.0 or higher) - Optionally [Docker](https://www.docker.com/) ### Locally diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 052f0e12d9..0e894d1c70 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -621,11 +621,11 @@ const buildWhereClause = ({ const distance = fieldValue.$maxDistance; const distanceInKM = distance * 6371 * 1000; patterns.push( - `ST_distance_sphere($${index}:name::geometry, POINT($${index + + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) <= $${index + 3}` ); sorts.push( - `ST_distance_sphere($${index}:name::geometry, POINT($${index + + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) ASC` ); values.push(fieldName, point.longitude, point.latitude, distanceInKM); @@ -673,7 +673,7 @@ const buildWhereClause = ({ } const distanceInKM = distance * 6371 * 1000; patterns.push( - `ST_distance_sphere($${index}:name::geometry, POINT($${index + + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) <= $${index + 3}` ); values.push(fieldName, point.longitude, point.latitude, distanceInKM); From 66de0265d6842248b251026cc67ddd6a26a972f0 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 13:00:58 -0400 Subject: [PATCH 09/50] updated postgres sections of contributions with newer postgres info. Also switched postgis image it points to as the other one hasn't been updated in over a year. --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 756a2efeab..c42296f759 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ Once you have babel running in watch mode, you can start making changes to parse If your pull request introduces a change that may affect the storage or retrieval of objects, you may want to make sure it plays nice with Postgres. -* Run the tests against the postgres database with `PARSE_SERVER_TEST_DB=postgres npm test`. You'll need to have postgres running on your machine and setup [appropriately](https://github.com/parse-community/parse-server/blob/master/.travis.yml#L37) or use [`Docker`](#run-a-parse-postgres-with-docker). +* Run the tests against the postgres database with `PARSE_SERVER_TEST_DB=postgres npm test`. You'll need to have postgres running on your machine and your configuratoin file setup [appropriately](https://github.com/postgres/postgres/blob/master/src/backend/utils/misc/postgresql.conf.sample#L63) along with [PostGIS](https://postgis.net) 2.2.0 higher installed, or use [`Docker`](#run-a-parse-postgres-with-docker). * The Postgres adapter has a special debugger that traces all the sql commands. You can enable it with setting the environment variable `PARSE_SERVER_LOG_LEVEL=debug` * If your feature is intended to only work with MongoDB, you should disable PostgreSQL-specific tests with: @@ -68,10 +68,10 @@ If your pull request introduces a change that may affect the storage or retrieva #### Run a Parse Postgres with Docker -To launch the compatible Postgres instance, copy and paste the following line into your shell: +[PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm mdillon/postgis:11-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: From 9faacb8105ebc5f31e48b7fb945a5ca871ec5d77 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 13:42:46 -0400 Subject: [PATCH 10/50] more info about postgres --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c42296f759..47f58394ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ If your pull request introduces a change that may affect the storage or retrieva #### Run a Parse Postgres with Docker -[PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: +[PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database From b93083b59daa7de8902f23642dd411f80baf1440 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 14:23:56 -0400 Subject: [PATCH 11/50] added necessary password for postgres docker --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 47f58394ef..96940c5c42 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: From d8417c1feb35a9b95eabd8666c28bf9916684e5d Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 14:44:55 -0400 Subject: [PATCH 12/50] updated wording in contributions --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96940c5c42..5c7b618e92 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,7 @@ If your pull request introduces a change that may affect the storage or retrieva - `it_only_db('mongo')` // will make a test that only runs on mongo - `it_exclude_dbs(['postgres'])` // will make a test that runs against all DB's but postgres -#### Run a Parse Postgres with Docker +#### Run Postgres setup for Parse with Docker [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: From 117ec8bde02545d49fa92dc1b86e395064023f52 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 14:54:13 -0400 Subject: [PATCH 13/50] removed reference to MacJr environment var when starting postgres in contributions. The official image automatically creates a user named 'postgres', but it does require a password, which the command sets to 'postgres' --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c7b618e92..d959be56a9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: From 2cf865d60051f050d7ab4b7741a0534ef99bad2e Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 15:22:36 -0400 Subject: [PATCH 14/50] added more time to docker sleep/wait to enter postgis commands. This will always take a few seconds because the db is installing from scratch everytime. If postgres/postgis images aren't already downloaded locally, it will take even longer. Worst case, if the command times out on first run. Stop and remove the parse-postgres container and run the command again, 20 seconds should be enough wait time then --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d959be56a9..893cf33b0f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 20 && docker exec -it parse-postgres psql -U $USER -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: From 6328b4f58bfa10a2b6506e90a11629e967f38d5d Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sat, 21 Mar 2020 10:56:22 -0400 Subject: [PATCH 15/50] latest changes --- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 0e894d1c70..5805a02fba 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2532,11 +2532,16 @@ export class PostgresStorageAdapter implements StorageAdapter { caseInsensitive: boolean = false ): Promise { + const indexCreationRequest = {}; + postgresFieldNames.forEach(fieldName => { + indexCreationRequest[fieldName] = 1; + }); + return this.setIndexesWithSchemaFormat( className, schema.indexes, - {}, - schema.fields, + schema.indexes, + indexCreationRequest, null ); //return Promise.resolve(); From cb096672852fedc766d989dffc54a669781974e8 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sat, 21 Mar 2020 17:13:50 -0400 Subject: [PATCH 16/50] initial fix, need to test --- package-lock.json | 17 ++++++-- package.json | 1 + spec/PostgresStorageAdapter.spec.js | 14 +++++-- .../Postgres/PostgresStorageAdapter.js | 42 ++++++++++++++----- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index f9d57d23e8..e7ce01c26f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7130,6 +7130,11 @@ "sshpk": "^1.7.0" } }, + "http2": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/http2/-/http2-3.3.7.tgz", + "integrity": "sha512-puSi8M8WNlFJm9Pk4c/Mbz9Gwparuj3gO9/RRO5zv6piQ0FY+9Qywp0PdWshYgsMJSalixFY7eC6oPu0zRxLAQ==" + }, "https-proxy-agent": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", @@ -9639,9 +9644,9 @@ } }, "nopt": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", - "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", "optional": true, "requires": { "abbrev": "1", @@ -10136,6 +10141,7 @@ "@babel/runtime-corejs3": "7.7.7", "crypto-js": "3.1.9-1", "uuid": "3.3.3", + "ws": "7.2.1", "xmlhttprequest": "1.8.0" }, "dependencies": { @@ -10160,6 +10166,11 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" + }, + "ws": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz", + "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A==" } } }, diff --git a/package.json b/package.json index 6d9c57d46b..dbc42ed72d 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "graphql-relay": "^0.6.0", "graphql-tools": "^4.0.7", "graphql-upload": "10.0.0", + "http2": "^3.3.7", "intersect": "1.0.1", "jsonwebtoken": "8.5.1", "jwks-rsa": "1.7.0", diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index 9658f4f572..a7f6016be4 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -161,8 +161,11 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { const database = Config.get(Parse.applicationId).database; const client = adapter._client; + + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's + await client.none('INSERT INTO "_User" (username, "objectId") SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)'); - const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower('bugs')') + const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); const schema = await new Parse.Schema('_User').get(); @@ -173,11 +176,14 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { 'case_insensitive_username', true ); + + const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); - const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower('bugs')') + //Delete generated data in postgres + await client.none('DELETE FROM "_User" WHERE "emailVerified" is null'); - expect(preIndexPlan.executionStats.executionStages.stage).toBe('COLLSCAN'); - expect(postIndexPlan.executionStats.executionStages.stage).toBe('FETCH'); + expect(preIndexPlan[0].Plan['Node Type']).toBe('Seq Scan'); + expect(postIndexPlan[0].Plan['Node Type']).not.toContain('Seq Scan'); }); }); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 5805a02fba..476ed1c709 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2523,7 +2523,6 @@ export class PostgresStorageAdapter implements StorageAdapter { return result; } - // TODO: implement? ensureIndex( className: string, schema: SchemaType, @@ -2532,18 +2531,39 @@ export class PostgresStorageAdapter implements StorageAdapter { caseInsensitive: boolean = false ): Promise { - const indexCreationRequest = {}; - postgresFieldNames.forEach(fieldName => { - indexCreationRequest[fieldName] = 1; - }); - return this.setIndexesWithSchemaFormat( - className, - schema.indexes, - schema.indexes, - indexCreationRequest, - null + const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; + const indexNameOptions: Object = indexName ? { name: indexName } : { name: defaultIndexName }; + + const constraintPatterns = caseInsensitive ? fieldNames.map( + (fieldName, index) => `lower($${index + 3}:name)` + ) : fieldNames.map( + (fieldName, index) => `$${index + 3}:name` ); + + const qs = caseInsensitive ? `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()}) varchar_pattern_ops)` : `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; + + return this._client + .none(qs, [indexName, className, ...fieldNames]) + .catch(error => { + if ( + error.code === PostgresDuplicateRelationError && + error.message.includes(indexNameOptions) + ) { + // Index already exists. Ignore error. + } else if ( + error.code === PostgresUniqueIndexViolationError && + error.message.includes(indexNameOptions) + ) { + // Cast the error into the proper parse error + throw new Parse.Error( + Parse.Error.DUPLICATE_VALUE, + 'A duplicate value for a field with unique values was provided' + ); + } else { + throw error;// + } + }); //return Promise.resolve(); } } From f0335c420cc2b8174dadc1d352d3a09a9e8816da Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sat, 21 Mar 2020 19:00:29 -0400 Subject: [PATCH 17/50] fixed lint --- spec/PostgresStorageAdapter.spec.js | 11 +---------- .../Storage/Postgres/PostgresStorageAdapter.js | 6 ------ 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index a7f6016be4..b383b7490a 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -151,24 +151,18 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { undefined ); }); - + it('should use index for caseInsensitive query', async () => { const user = new Parse.User(); user.set('username', 'Bugs'); user.set('password', 'Bunny'); await user.signUp(); - const database = Config.get(Parse.applicationId).database; - const client = adapter._client; - //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's await client.none('INSERT INTO "_User" (username, "objectId") SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)'); - const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); - const schema = await new Parse.Schema('_User').get(); - await database.adapter.ensureIndex( '_User', schema, @@ -176,12 +170,9 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { 'case_insensitive_username', true ); - const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); - //Delete generated data in postgres await client.none('DELETE FROM "_User" WHERE "emailVerified" is null'); - expect(preIndexPlan[0].Plan['Node Type']).toBe('Seq Scan'); expect(postIndexPlan[0].Plan['Node Type']).not.toContain('Seq Scan'); }); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 476ed1c709..ceb781a0e0 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2530,19 +2530,14 @@ export class PostgresStorageAdapter implements StorageAdapter { indexName: ?string, caseInsensitive: boolean = false ): Promise { - - const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; const indexNameOptions: Object = indexName ? { name: indexName } : { name: defaultIndexName }; - const constraintPatterns = caseInsensitive ? fieldNames.map( (fieldName, index) => `lower($${index + 3}:name)` ) : fieldNames.map( (fieldName, index) => `$${index + 3}:name` ); - const qs = caseInsensitive ? `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()}) varchar_pattern_ops)` : `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; - return this._client .none(qs, [indexName, className, ...fieldNames]) .catch(error => { @@ -2564,7 +2559,6 @@ export class PostgresStorageAdapter implements StorageAdapter { throw error;// } }); - //return Promise.resolve(); } } From 06a74c93eff8b9fcde2bbad1884e4baba6d1c65f Mon Sep 17 00:00:00 2001 From: Corey Date: Thu, 12 Mar 2020 20:56:02 -0400 Subject: [PATCH 18/50] Adding test case for caseInsensitive Adding test case for verifying indexing for caseInsensitive --- spec/PostgresStorageAdapter.spec.js | 35 +++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index e722fa4cec..9658f4f572 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -16,6 +16,13 @@ const dropTable = (client, className) => { return client.none('DROP TABLE IF EXISTS $', { className }); }; +const getQueryPlan = (client, query) => { + return client.any( + 'EXPLAIN (ANALYZE, FORMAT JSON) $', + { query } + ); +}; + describe_only_db('postgres')('PostgresStorageAdapter', () => { let adapter; beforeEach(() => { @@ -144,6 +151,34 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { undefined ); }); + + it('should use index for caseInsensitive query', async () => { + const user = new Parse.User(); + user.set('username', 'Bugs'); + user.set('password', 'Bunny'); + await user.signUp(); + + const database = Config.get(Parse.applicationId).database; + + const client = adapter._client; + + const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower('bugs')') + + const schema = await new Parse.Schema('_User').get(); + + await database.adapter.ensureIndex( + '_User', + schema, + ['username'], + 'case_insensitive_username', + true + ); + + const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower('bugs')') + + expect(preIndexPlan.executionStats.executionStages.stage).toBe('COLLSCAN'); + expect(postIndexPlan.executionStats.executionStages.stage).toBe('FETCH'); + }); }); describe_only_db('postgres')('PostgresStorageAdapter shutdown', () => { From 72ad0cde95a328fe7c364ad70447da223c413e15 Mon Sep 17 00:00:00 2001 From: Corey Date: Thu, 12 Mar 2020 20:57:06 -0400 Subject: [PATCH 19/50] Implementing ensureIndex --- .../Storage/Postgres/PostgresStorageAdapter.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 31602afc0d..052f0e12d9 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2524,8 +2524,22 @@ export class PostgresStorageAdapter implements StorageAdapter { } // TODO: implement? - ensureIndex(): Promise { - return Promise.resolve(); + ensureIndex( + className: string, + schema: SchemaType, + fieldNames: string[], + indexName: ?string, + caseInsensitive: boolean = false + ): Promise { + + return this.setIndexesWithSchemaFormat( + className, + schema.indexes, + {}, + schema.fields, + null + ); + //return Promise.resolve(); } } From 0bdd53c15dd9740120a5c5972b69071996bf6fb4 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 12:18:31 -0400 Subject: [PATCH 20/50] Updated PostgresStorageAdapter calls to ST_DistanceSphere. Note this has a minimum requirement of postgis 2.2. Documented the change in the readme. This is address #6441 --- README.md | 2 +- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b935b83e03..b5e5d43d26 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ The fastest and easiest way to get started is to run MongoDB and Parse Server lo Before you start make sure you have installed: - [NodeJS](https://www.npmjs.com/) that includes `npm` -- [MongoDB](https://www.mongodb.com/) or [PostgreSQL](https://www.postgresql.org/) +- [MongoDB](https://www.mongodb.com/) or [PostgreSQL](https://www.postgresql.org/)(with [PostGIS](https://postgis.net) 2.2.0 or higher) - Optionally [Docker](https://www.docker.com/) ### Locally diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 052f0e12d9..0e894d1c70 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -621,11 +621,11 @@ const buildWhereClause = ({ const distance = fieldValue.$maxDistance; const distanceInKM = distance * 6371 * 1000; patterns.push( - `ST_distance_sphere($${index}:name::geometry, POINT($${index + + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) <= $${index + 3}` ); sorts.push( - `ST_distance_sphere($${index}:name::geometry, POINT($${index + + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) ASC` ); values.push(fieldName, point.longitude, point.latitude, distanceInKM); @@ -673,7 +673,7 @@ const buildWhereClause = ({ } const distanceInKM = distance * 6371 * 1000; patterns.push( - `ST_distance_sphere($${index}:name::geometry, POINT($${index + + `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) <= $${index + 3}` ); values.push(fieldName, point.longitude, point.latitude, distanceInKM); From 2b7d6f4d9ce861f15cbb312c123ded678d1dc685 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 13:00:58 -0400 Subject: [PATCH 21/50] updated postgres sections of contributions with newer postgres info. Also switched postgis image it points to as the other one hasn't been updated in over a year. --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 756a2efeab..c42296f759 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ Once you have babel running in watch mode, you can start making changes to parse If your pull request introduces a change that may affect the storage or retrieval of objects, you may want to make sure it plays nice with Postgres. -* Run the tests against the postgres database with `PARSE_SERVER_TEST_DB=postgres npm test`. You'll need to have postgres running on your machine and setup [appropriately](https://github.com/parse-community/parse-server/blob/master/.travis.yml#L37) or use [`Docker`](#run-a-parse-postgres-with-docker). +* Run the tests against the postgres database with `PARSE_SERVER_TEST_DB=postgres npm test`. You'll need to have postgres running on your machine and your configuratoin file setup [appropriately](https://github.com/postgres/postgres/blob/master/src/backend/utils/misc/postgresql.conf.sample#L63) along with [PostGIS](https://postgis.net) 2.2.0 higher installed, or use [`Docker`](#run-a-parse-postgres-with-docker). * The Postgres adapter has a special debugger that traces all the sql commands. You can enable it with setting the environment variable `PARSE_SERVER_LOG_LEVEL=debug` * If your feature is intended to only work with MongoDB, you should disable PostgreSQL-specific tests with: @@ -68,10 +68,10 @@ If your pull request introduces a change that may affect the storage or retrieva #### Run a Parse Postgres with Docker -To launch the compatible Postgres instance, copy and paste the following line into your shell: +[PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm mdillon/postgis:11-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: From 4555a87a112eddbd80c1c8e4e236f2361693cea5 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 13:42:46 -0400 Subject: [PATCH 22/50] more info about postgres --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c42296f759..47f58394ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ If your pull request introduces a change that may affect the storage or retrieva #### Run a Parse Postgres with Docker -[PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: +[PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database From c9d8680e7330bacf6a8fb3cbd99d01b1d3d172e4 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 14:23:56 -0400 Subject: [PATCH 23/50] added necessary password for postgres docker --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 47f58394ef..96940c5c42 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: From 63c25e06819446bbf60b5d5636fea8b9f86788ac Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 14:44:55 -0400 Subject: [PATCH 24/50] updated wording in contributions --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96940c5c42..5c7b618e92 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,7 @@ If your pull request introduces a change that may affect the storage or retrieva - `it_only_db('mongo')` // will make a test that only runs on mongo - `it_exclude_dbs(['postgres'])` // will make a test that runs against all DB's but postgres -#### Run a Parse Postgres with Docker +#### Run Postgres setup for Parse with Docker [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: From 1d078f937fb2e56220c4c529c904fc5ecc0f245c Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 14:54:13 -0400 Subject: [PATCH 25/50] removed reference to MacJr environment var when starting postgres in contributions. The official image automatically creates a user named 'postgres', but it does require a password, which the command sets to 'postgres' --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c7b618e92..d959be56a9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: From e1f86c8ccc0dba1c845e8f9632685465f704170b Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sun, 15 Mar 2020 15:22:36 -0400 Subject: [PATCH 26/50] added more time to docker sleep/wait to enter postgis commands. This will always take a few seconds because the db is installing from scratch everytime. If postgres/postgis images aren't already downloaded locally, it will take even longer. Worst case, if the command times out on first run. Stop and remove the parse-postgres container and run the command again, 20 seconds should be enough wait time then --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d959be56a9..893cf33b0f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 5 && docker exec -it parse-postgres psql -U $USER -c 'create database parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 20 && docker exec -it parse-postgres psql -U $USER -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: From 297426b062cf0b41dbacdcfac98bd594d25952ce Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sat, 21 Mar 2020 10:56:22 -0400 Subject: [PATCH 27/50] latest changes --- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 0e894d1c70..5805a02fba 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2532,11 +2532,16 @@ export class PostgresStorageAdapter implements StorageAdapter { caseInsensitive: boolean = false ): Promise { + const indexCreationRequest = {}; + postgresFieldNames.forEach(fieldName => { + indexCreationRequest[fieldName] = 1; + }); + return this.setIndexesWithSchemaFormat( className, schema.indexes, - {}, - schema.fields, + schema.indexes, + indexCreationRequest, null ); //return Promise.resolve(); From e59284cec811a80f4d57a7f09d3ea1127055076b Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sat, 21 Mar 2020 17:13:50 -0400 Subject: [PATCH 28/50] initial fix, need to test --- package-lock.json | 17 ++++++-- package.json | 1 + spec/PostgresStorageAdapter.spec.js | 14 +++++-- .../Postgres/PostgresStorageAdapter.js | 42 ++++++++++++++----- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/package-lock.json b/package-lock.json index ca98da74d9..1f86f7a1ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7172,6 +7172,11 @@ "sshpk": "^1.7.0" } }, + "http2": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/http2/-/http2-3.3.7.tgz", + "integrity": "sha512-puSi8M8WNlFJm9Pk4c/Mbz9Gwparuj3gO9/RRO5zv6piQ0FY+9Qywp0PdWshYgsMJSalixFY7eC6oPu0zRxLAQ==" + }, "https-proxy-agent": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", @@ -9688,9 +9693,9 @@ } }, "nopt": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", - "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", "optional": true, "requires": { "abbrev": "1", @@ -10185,6 +10190,7 @@ "@babel/runtime-corejs3": "7.7.7", "crypto-js": "3.1.9-1", "uuid": "3.3.3", + "ws": "7.2.1", "xmlhttprequest": "1.8.0" }, "dependencies": { @@ -10209,6 +10215,11 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" + }, + "ws": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz", + "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A==" } } }, diff --git a/package.json b/package.json index 25991a4351..7b86b7b05d 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "graphql-relay": "^0.6.0", "graphql-tools": "^4.0.7", "graphql-upload": "10.0.0", + "http2": "^3.3.7", "intersect": "1.0.1", "jsonwebtoken": "8.5.1", "jwks-rsa": "1.7.0", diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index 9658f4f572..a7f6016be4 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -161,8 +161,11 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { const database = Config.get(Parse.applicationId).database; const client = adapter._client; + + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's + await client.none('INSERT INTO "_User" (username, "objectId") SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)'); - const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower('bugs')') + const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); const schema = await new Parse.Schema('_User').get(); @@ -173,11 +176,14 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { 'case_insensitive_username', true ); + + const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); - const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower('bugs')') + //Delete generated data in postgres + await client.none('DELETE FROM "_User" WHERE "emailVerified" is null'); - expect(preIndexPlan.executionStats.executionStages.stage).toBe('COLLSCAN'); - expect(postIndexPlan.executionStats.executionStages.stage).toBe('FETCH'); + expect(preIndexPlan[0].Plan['Node Type']).toBe('Seq Scan'); + expect(postIndexPlan[0].Plan['Node Type']).not.toContain('Seq Scan'); }); }); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 5805a02fba..476ed1c709 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2523,7 +2523,6 @@ export class PostgresStorageAdapter implements StorageAdapter { return result; } - // TODO: implement? ensureIndex( className: string, schema: SchemaType, @@ -2532,18 +2531,39 @@ export class PostgresStorageAdapter implements StorageAdapter { caseInsensitive: boolean = false ): Promise { - const indexCreationRequest = {}; - postgresFieldNames.forEach(fieldName => { - indexCreationRequest[fieldName] = 1; - }); - return this.setIndexesWithSchemaFormat( - className, - schema.indexes, - schema.indexes, - indexCreationRequest, - null + const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; + const indexNameOptions: Object = indexName ? { name: indexName } : { name: defaultIndexName }; + + const constraintPatterns = caseInsensitive ? fieldNames.map( + (fieldName, index) => `lower($${index + 3}:name)` + ) : fieldNames.map( + (fieldName, index) => `$${index + 3}:name` ); + + const qs = caseInsensitive ? `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()}) varchar_pattern_ops)` : `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; + + return this._client + .none(qs, [indexName, className, ...fieldNames]) + .catch(error => { + if ( + error.code === PostgresDuplicateRelationError && + error.message.includes(indexNameOptions) + ) { + // Index already exists. Ignore error. + } else if ( + error.code === PostgresUniqueIndexViolationError && + error.message.includes(indexNameOptions) + ) { + // Cast the error into the proper parse error + throw new Parse.Error( + Parse.Error.DUPLICATE_VALUE, + 'A duplicate value for a field with unique values was provided' + ); + } else { + throw error;// + } + }); //return Promise.resolve(); } } From 395b9dc48aec43c8548437abb8be16dd4ee80131 Mon Sep 17 00:00:00 2001 From: Corey MacBook Pro Date: Sat, 21 Mar 2020 19:00:29 -0400 Subject: [PATCH 29/50] fixed lint --- spec/PostgresStorageAdapter.spec.js | 11 +---------- .../Storage/Postgres/PostgresStorageAdapter.js | 6 ------ 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index a7f6016be4..b383b7490a 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -151,24 +151,18 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { undefined ); }); - + it('should use index for caseInsensitive query', async () => { const user = new Parse.User(); user.set('username', 'Bugs'); user.set('password', 'Bunny'); await user.signUp(); - const database = Config.get(Parse.applicationId).database; - const client = adapter._client; - //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's await client.none('INSERT INTO "_User" (username, "objectId") SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)'); - const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); - const schema = await new Parse.Schema('_User').get(); - await database.adapter.ensureIndex( '_User', schema, @@ -176,12 +170,9 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { 'case_insensitive_username', true ); - const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); - //Delete generated data in postgres await client.none('DELETE FROM "_User" WHERE "emailVerified" is null'); - expect(preIndexPlan[0].Plan['Node Type']).toBe('Seq Scan'); expect(postIndexPlan[0].Plan['Node Type']).not.toContain('Seq Scan'); }); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 476ed1c709..ceb781a0e0 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2530,19 +2530,14 @@ export class PostgresStorageAdapter implements StorageAdapter { indexName: ?string, caseInsensitive: boolean = false ): Promise { - - const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; const indexNameOptions: Object = indexName ? { name: indexName } : { name: defaultIndexName }; - const constraintPatterns = caseInsensitive ? fieldNames.map( (fieldName, index) => `lower($${index + 3}:name)` ) : fieldNames.map( (fieldName, index) => `$${index + 3}:name` ); - const qs = caseInsensitive ? `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()}) varchar_pattern_ops)` : `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; - return this._client .none(qs, [indexName, className, ...fieldNames]) .catch(error => { @@ -2564,7 +2559,6 @@ export class PostgresStorageAdapter implements StorageAdapter { throw error;// } }); - //return Promise.resolve(); } } From c824809f617228c894f0ac508447cd55cf51805d Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 00:50:39 -0400 Subject: [PATCH 30/50] Adds caseInsensitive constraints to database, but doesn't pass regular tests. I believe this is because ensureIndex in the Postgres adapter is returning wrong. Also, some issues with the caseInsensitive test case --- package-lock.json | 14 ++++++++++++++ package.json | 1 - spec/PostgresStorageAdapter.spec.js | 25 +++++++++++++++---------- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1f86f7a1ce..16a297a0bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,6 +51,7 @@ "dev": true, "requires": { "chokidar": "^2.1.8", + "commander": "^4.0.1", "convert-source-map": "^1.1.0", "fs-readdir-recursive": "^1.1.0", "glob": "^7.0.0", @@ -60,6 +61,12 @@ "source-map": "^0.5.0" }, "dependencies": { + "commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true + }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -8285,6 +8292,7 @@ "dev": true, "requires": { "chalk": "^3.0.0", + "commander": "^4.0.1", "cosmiconfig": "^6.0.0", "debug": "^4.1.1", "dedent": "^0.7.0", @@ -8342,6 +8350,12 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true + }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", diff --git a/package.json b/package.json index 7b86b7b05d..25991a4351 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "graphql-relay": "^0.6.0", "graphql-tools": "^4.0.7", "graphql-upload": "10.0.0", - "http2": "^3.3.7", "intersect": "1.0.1", "jsonwebtoken": "8.5.1", "jwks-rsa": "1.7.0", diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index b383b7490a..e0d1254ccb 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -12,15 +12,12 @@ const getColumns = (client, className) => { ); }; -const dropTable = (client, className) => { - return client.none('DROP TABLE IF EXISTS $', { className }); +const getQueryPlan = (client, query) => { + return client.none('EXPLAIN (ANALYZE, FORMAT JSON) $', { query }); }; -const getQueryPlan = (client, query) => { - return client.any( - 'EXPLAIN (ANALYZE, FORMAT JSON) $', - { query } - ); +const dropTable = (client, className) => { + return client.none('DROP TABLE IF EXISTS $', { className }); }; describe_only_db('postgres')('PostgresStorageAdapter', () => { @@ -160,8 +157,13 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { const database = Config.get(Parse.applicationId).database; const client = adapter._client; //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's - await client.none('INSERT INTO "_User" (username, "objectId") SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)'); - const preIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); + await client.none( + 'INSERT INTO "_User" (username, "objectId") SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)' + ); //This isn't creanting what's its suppose to + const preIndexPlan = getQueryPlan( + client, + 'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')' + ); //There's an error here, ' isn't escaping properly const schema = await new Parse.Schema('_User').get(); await database.adapter.ensureIndex( '_User', @@ -170,7 +172,10 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { 'case_insensitive_username', true ); - const postIndexPlan = getQueryPlan(client,'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')'); + const postIndexPlan = getQueryPlan( + client, + 'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')' + ); //Delete generated data in postgres await client.none('DELETE FROM "_User" WHERE "emailVerified" is null'); expect(preIndexPlan[0].Plan['Node Type']).toBe('Seq Scan'); From 72314478374e083dbff270f7af333287ea599815 Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 02:01:22 -0400 Subject: [PATCH 31/50] this version addes the indexes, but something still wrong with the ensureIndex method in adapter --- .../Storage/Postgres/PostgresStorageAdapter.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index ceb781a0e0..705a00d47f 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2523,23 +2523,22 @@ export class PostgresStorageAdapter implements StorageAdapter { return result; } - ensureIndex( + async ensureIndex( className: string, schema: SchemaType, fieldNames: string[], indexName: ?string, caseInsensitive: boolean = false - ): Promise { + ): Promise { const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; const indexNameOptions: Object = indexName ? { name: indexName } : { name: defaultIndexName }; const constraintPatterns = caseInsensitive ? fieldNames.map( - (fieldName, index) => `lower($${index + 3}:name)` + (fieldName, index) => `lower($${index + 3}:name) varchar_pattern_ops` ) : fieldNames.map( (fieldName, index) => `$${index + 3}:name` ); - const qs = caseInsensitive ? `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()}) varchar_pattern_ops)` : `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; - return this._client - .none(qs, [indexName, className, ...fieldNames]) + const qs = `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; + return this._client.none(qs, [indexName, className, ...fieldNames]) .catch(error => { if ( error.code === PostgresDuplicateRelationError && @@ -2556,7 +2555,7 @@ export class PostgresStorageAdapter implements StorageAdapter { 'A duplicate value for a field with unique values was provided' ); } else { - throw error;// + throw error; } }); } From 323021be13e5c52450ce73554be4420c3c5fef90 Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 02:07:58 -0400 Subject: [PATCH 32/50] removed code from suggestions --- package-lock.json | 5 ----- package.json | 1 - src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 6 +++--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 16a297a0bc..723676fc24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7179,11 +7179,6 @@ "sshpk": "^1.7.0" } }, - "http2": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/http2/-/http2-3.3.7.tgz", - "integrity": "sha512-puSi8M8WNlFJm9Pk4c/Mbz9Gwparuj3gO9/RRO5zv6piQ0FY+9Qywp0PdWshYgsMJSalixFY7eC6oPu0zRxLAQ==" - }, "https-proxy-agent": { "version": "2.2.4", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz", diff --git a/package.json b/package.json index 7b86b7b05d..25991a4351 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "graphql-relay": "^0.6.0", "graphql-tools": "^4.0.7", "graphql-upload": "10.0.0", - "http2": "^3.3.7", "intersect": "1.0.1", "jsonwebtoken": "8.5.1", "jwks-rsa": "1.7.0", diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 705a00d47f..cb98b92413 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -621,11 +621,11 @@ const buildWhereClause = ({ const distance = fieldValue.$maxDistance; const distanceInKM = distance * 6371 * 1000; patterns.push( - `ST_DistanceSphere($${index}:name::geometry, POINT($${index + + `ST_distance_sphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) <= $${index + 3}` ); sorts.push( - `ST_DistanceSphere($${index}:name::geometry, POINT($${index + + `ST_distance_sphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) ASC` ); values.push(fieldName, point.longitude, point.latitude, distanceInKM); @@ -673,7 +673,7 @@ const buildWhereClause = ({ } const distanceInKM = distance * 6371 * 1000; patterns.push( - `ST_DistanceSphere($${index}:name::geometry, POINT($${index + + `ST_distance_sphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2})::geometry) <= $${index + 3}` ); values.push(fieldName, point.longitude, point.latitude, distanceInKM); From 0812679cb67a1981ae71c485428c7b9801ac6eff Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 13:02:23 -0400 Subject: [PATCH 33/50] fixed lint --- spec/PostgresStorageAdapter.spec.js | 77 +++++++++++-------- .../Postgres/PostgresStorageAdapter.js | 13 ++-- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index e0d1254ccb..40676efd4a 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -12,8 +12,8 @@ const getColumns = (client, className) => { ); }; -const getQueryPlan = (client, query) => { - return client.none('EXPLAIN (ANALYZE, FORMAT JSON) $', { query }); +const createExplainableQuery = (client, query) => { + return 'EXPLAIN (ANALYZE, FORMAT JSON) ' + query; }; const dropTable = (client, className) => { @@ -150,36 +150,51 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { }); it('should use index for caseInsensitive query', async () => { - const user = new Parse.User(); - user.set('username', 'Bugs'); - user.set('password', 'Bunny'); - await user.signUp(); - const database = Config.get(Parse.applicationId).database; + + const tableName = 'MyCaseSensitiveTable'; + const schema = { + fields: { + columnA: { type: 'String' }, + columnB: { type: 'String' }, + columnC: { type: 'String' }, + }, + }; + + await adapter.createTable(tableName, schema); const client = adapter._client; - //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's - await client.none( - 'INSERT INTO "_User" (username, "objectId") SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)' - ); //This isn't creanting what's its suppose to - const preIndexPlan = getQueryPlan( - client, - 'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')' - ); //There's an error here, ' isn't escaping properly - const schema = await new Parse.Schema('_User').get(); - await database.adapter.ensureIndex( - '_User', - schema, - ['username'], - 'case_insensitive_username', - true - ); - const postIndexPlan = getQueryPlan( - client, - 'SELECT * FROM "_User" WHERE lower(username)=lower(\'bugs\')' - ); - //Delete generated data in postgres - await client.none('DELETE FROM "_User" WHERE "emailVerified" is null'); - expect(preIndexPlan[0].Plan['Node Type']).toBe('Seq Scan'); - expect(postIndexPlan[0].Plan['Node Type']).not.toContain('Seq Scan'); + + client.none('INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', [tableName, 'columnA', 'columnB', 'Bugs', 'Bunny']) + .then(() => { + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's + client.none( + 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)', [tableName, 'columnA' , 'columnB'] + ) + .then(() => { + const caseInsensitiveData = 'bugs'; + const qs = createExplainableQuery(client, 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)'); + client.one(qs, [tableName, 'columnA', caseInsensitiveData]) + .then(explained => { + //expect(1).toBe(1); + //expect(explained['QUERY PLAN']).toBe('Seq Scan'); + const test = explained['QUERY PLAN']; + const test2 = test.Plan; + expect(test).toEqual(test2); + /*adapter.ensureIndex( + tableName, + schema, + ['columnB'], + 'test_case_insensitive_column', + true + ).then(() => { + client.one(qs, [tableName, 'columnA', caseInsensitiveData]).then(explained => { + expect(explained.Plan['Node Type']).not.toContain('Seq Scan'); + //Delete generated data in postgres + client.none('DELETE FROM $1:name WHERE $2:name is null', [tableName, 'columnC']); + }) + });*/ + }); + }); + }); }); }); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index cb98b92413..9c153e12d1 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2532,22 +2532,19 @@ export class PostgresStorageAdapter implements StorageAdapter { ): Promise { const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; const indexNameOptions: Object = indexName ? { name: indexName } : { name: defaultIndexName }; - const constraintPatterns = caseInsensitive ? fieldNames.map( - (fieldName, index) => `lower($${index + 3}:name) varchar_pattern_ops` - ) : fieldNames.map( - (fieldName, index) => `$${index + 3}:name` - ); + const constraintPatterns = caseInsensitive ? fieldNames.map((fieldName, index) => `lower($${index + 3}:name) varchar_pattern_ops`) : + fieldNames.map((fieldName, index) => `$${index + 3}:name`); const qs = `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; - return this._client.none(qs, [indexName, className, ...fieldNames]) + await this._client.none(qs, [indexNameOptions.name, className, ...fieldNames]) .catch(error => { if ( error.code === PostgresDuplicateRelationError && - error.message.includes(indexNameOptions) + error.message.includes(indexNameOptions.name) ) { // Index already exists. Ignore error. } else if ( error.code === PostgresUniqueIndexViolationError && - error.message.includes(indexNameOptions) + error.message.includes(indexNameOptions.name) ) { // Cast the error into the proper parse error throw new Parse.Error( From 9f3f5da78babb8ff91b16266f18035b4fce752bd Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 14:28:15 -0400 Subject: [PATCH 34/50] fixed PostgresAdapter test case --- spec/PostgresStorageAdapter.spec.js | 79 ++++++++++++++++++----------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index 40676efd4a..fb413693b5 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -150,48 +150,67 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { }); it('should use index for caseInsensitive query', async () => { - - const tableName = 'MyCaseSensitiveTable'; + const tableName = '_User'; const schema = { fields: { - columnA: { type: 'String' }, - columnB: { type: 'String' }, - columnC: { type: 'String' }, + objectId: { type: 'String' }, + username: { type: 'String' }, + email: { type: 'String' }, }, }; await adapter.createTable(tableName, schema); const client = adapter._client; - client.none('INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', [tableName, 'columnA', 'columnB', 'Bugs', 'Bunny']) + client + .none('INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', [ + tableName, + 'objectId', + 'username', + 'Bugs', + 'Bunny', + ]) .then(() => { - //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's - client.none( - 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,11000)', [tableName, 'columnA' , 'columnB'] - ) + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's + client + .none( + 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)', + [tableName, 'objectId', 'username'] + ) .then(() => { const caseInsensitiveData = 'bugs'; - const qs = createExplainableQuery(client, 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)'); - client.one(qs, [tableName, 'columnA', caseInsensitiveData]) + const qs = createExplainableQuery( + client, + 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' + ); + client + .one(qs, [tableName, 'objectId', caseInsensitiveData]) .then(explained => { - //expect(1).toBe(1); - //expect(explained['QUERY PLAN']).toBe('Seq Scan'); - const test = explained['QUERY PLAN']; - const test2 = test.Plan; - expect(test).toEqual(test2); - /*adapter.ensureIndex( - tableName, - schema, - ['columnB'], - 'test_case_insensitive_column', - true - ).then(() => { - client.one(qs, [tableName, 'columnA', caseInsensitiveData]).then(explained => { - expect(explained.Plan['Node Type']).not.toContain('Seq Scan'); - //Delete generated data in postgres - client.none('DELETE FROM $1:name WHERE $2:name is null', [tableName, 'columnC']); - }) - });*/ + expect(explained['QUERY PLAN'][0].Plan['Node Type']).toBe( + 'Seq Scan' + ); + adapter + .ensureIndex( + tableName, + schema, + ['objectId'], + 'test_case_insensitive_column', + true + ) + .then(() => { + client + .one(qs, [tableName, 'objectId', caseInsensitiveData]) + .then(explained => { + expect( + explained['QUERY PLAN'][0].Plan['Node Type'] + ).not.toContain('Seq Scan'); + //Delete generated data in postgres + client.none( + 'DELETE FROM $1:name WHERE $2:name is null', + [tableName, 'email'] + ); + }); + }); }); }); }); From c9dcab6530d08beda948b84ecd8d77691e9962ac Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 16:17:24 -0400 Subject: [PATCH 35/50] small bug in test case --- spec/PostgresStorageAdapter.spec.js | 77 +++++++++++++++++-- .../Postgres/PostgresStorageAdapter.js | 2 +- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index fb413693b5..e605915678 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -89,6 +89,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { expect(columns).toContain('columnA'); expect(columns).toContain('columnB'); expect(columns).toContain('columnC'); + dropTable(client, className); done(); }) .catch(error => done.fail(error)); @@ -117,6 +118,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { expect(columns.length).toEqual(2); expect(columns).toContain('columnA'); expect(columns).toContain('columnB'); + dropTable(client, className); done(); }) .catch(done); @@ -158,9 +160,9 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { email: { type: 'String' }, }, }; - - await adapter.createTable(tableName, schema); const client = adapter._client; + await dropTable(client, tableName); + await adapter.createTable(tableName, schema); client .none('INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', [ @@ -189,12 +191,13 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { expect(explained['QUERY PLAN'][0].Plan['Node Type']).toBe( 'Seq Scan' ); + const indexName = 'test_case_insensitive_column' adapter .ensureIndex( tableName, schema, ['objectId'], - 'test_case_insensitive_column', + indexName, true ) .then(() => { @@ -204,17 +207,77 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { expect( explained['QUERY PLAN'][0].Plan['Node Type'] ).not.toContain('Seq Scan'); + expect( + explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] + ).toBe(indexName); //Delete generated data in postgres - client.none( - 'DELETE FROM $1:name WHERE $2:name is null', - [tableName, 'email'] - ); + dropTable(client, tableName); }); }); }); }); }); }); + + it('should use index for caseInsensitive query using default indexname', async () => { + const tableName = 'CaseTable'; + const schema = { + fields: { + objectId: { type: 'String' }, + username: { type: 'String' }, + email: { type: 'String' }, + }, + }; + const client = adapter._client; + await dropTable(client, tableName); + await adapter.createTable(tableName, schema); + + client + .none('INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', [ + tableName, + 'objectId', + 'username', + 'Bugs', + 'Bunny', + ]) + .then(() => { + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's + client + .none( + 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)', + [tableName, 'objectId', 'username'] + ) + .then(() => { + const caseInsensitiveData = 'bugs'; + const qs = createExplainableQuery( + client, + 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' + ); + adapter + .ensureIndex( + tableName, + schema, + ['objectId'], + null, + true + ) + .then(() => { + client + .one(qs, [tableName, 'objectId', caseInsensitiveData]) + .then(explained => { + expect( + explained['QUERY PLAN'][0].Plan['Node Type'] + ).not.toContain('Seq Scan'); + expect( + explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] + ).toContain('parse_default'); + //Delete generated data in postgres + dropTable(client, tableName); + }); + }); + }); + }); + }); }); describe_only_db('postgres')('PostgresStorageAdapter shutdown', () => { diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 9c153e12d1..a88cb225c1 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2531,7 +2531,7 @@ export class PostgresStorageAdapter implements StorageAdapter { caseInsensitive: boolean = false ): Promise { const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; - const indexNameOptions: Object = indexName ? { name: indexName } : { name: defaultIndexName }; + const indexNameOptions: Object = indexName != null ? { name: indexName } : { name: defaultIndexName }; const constraintPatterns = caseInsensitive ? fieldNames.map((fieldName, index) => `lower($${index + 3}:name) varchar_pattern_ops`) : fieldNames.map((fieldName, index) => `$${index + 3}:name`); const qs = `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; From c7ba030c72e932c8ce7cc77fb56edd6f5f68c35a Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 16:53:11 -0400 Subject: [PATCH 36/50] reverted back to main branch package.json and lock file --- package-lock.json | 26 +---- spec/PostgresStorageAdapter.spec.js | 141 +++++++++++----------------- 2 files changed, 57 insertions(+), 110 deletions(-) diff --git a/package-lock.json b/package-lock.json index 723676fc24..ca98da74d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,7 +51,6 @@ "dev": true, "requires": { "chokidar": "^2.1.8", - "commander": "^4.0.1", "convert-source-map": "^1.1.0", "fs-readdir-recursive": "^1.1.0", "glob": "^7.0.0", @@ -61,12 +60,6 @@ "source-map": "^0.5.0" }, "dependencies": { - "commander": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", - "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true - }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -8287,7 +8280,6 @@ "dev": true, "requires": { "chalk": "^3.0.0", - "commander": "^4.0.1", "cosmiconfig": "^6.0.0", "debug": "^4.1.1", "dedent": "^0.7.0", @@ -8345,12 +8337,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, - "commander": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", - "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "dev": true - }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -9702,9 +9688,9 @@ } }, "nopt": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", - "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", + "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", "optional": true, "requires": { "abbrev": "1", @@ -10199,7 +10185,6 @@ "@babel/runtime-corejs3": "7.7.7", "crypto-js": "3.1.9-1", "uuid": "3.3.3", - "ws": "7.2.1", "xmlhttprequest": "1.8.0" }, "dependencies": { @@ -10224,11 +10209,6 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" - }, - "ws": { - "version": "7.2.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz", - "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A==" } } }, diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index e605915678..82c7d41cbc 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -163,57 +163,39 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { const client = adapter._client; await dropTable(client, tableName); await adapter.createTable(tableName, schema); - - client - .none('INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', [ - tableName, - 'objectId', - 'username', - 'Bugs', - 'Bunny', - ]) - .then(() => { - //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's - client - .none( - 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)', - [tableName, 'objectId', 'username'] - ) + await client.none( + 'INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', + [tableName, 'objectId', 'username', 'Bugs', 'Bunny'] + ); + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's + await client.none( + 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)', + [tableName, 'objectId', 'username'] + ); + const caseInsensitiveData = 'bugs'; + const qs = createExplainableQuery( + client, + 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' + ); + await client + .one(qs, [tableName, 'objectId', caseInsensitiveData]) + .then(explained => { + expect(explained['QUERY PLAN'][0].Plan['Node Type']).toBe('Seq Scan'); + const indexName = 'test_case_insensitive_column'; + adapter + .ensureIndex(tableName, schema, ['objectId'], indexName, true) .then(() => { - const caseInsensitiveData = 'bugs'; - const qs = createExplainableQuery( - client, - 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' - ); client .one(qs, [tableName, 'objectId', caseInsensitiveData]) .then(explained => { - expect(explained['QUERY PLAN'][0].Plan['Node Type']).toBe( - 'Seq Scan' - ); - const indexName = 'test_case_insensitive_column' - adapter - .ensureIndex( - tableName, - schema, - ['objectId'], - indexName, - true - ) - .then(() => { - client - .one(qs, [tableName, 'objectId', caseInsensitiveData]) - .then(explained => { - expect( - explained['QUERY PLAN'][0].Plan['Node Type'] - ).not.toContain('Seq Scan'); - expect( - explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] - ).toBe(indexName); - //Delete generated data in postgres - dropTable(client, tableName); - }); - }); + expect( + explained['QUERY PLAN'][0].Plan['Node Type'] + ).not.toContain('Seq Scan'); + expect( + explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] + ).toBe(indexName); + //Delete generated data in postgres + return dropTable(client, tableName); }); }); }); @@ -231,50 +213,35 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { const client = adapter._client; await dropTable(client, tableName); await adapter.createTable(tableName, schema); + await client.none( + 'INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', + [tableName, 'objectId', 'username', 'Bugs', 'Bunny'] + ); - client - .none('INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', [ - tableName, - 'objectId', - 'username', - 'Bugs', - 'Bunny', - ]) + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's + await client.none( + 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)', + [tableName, 'objectId', 'username'] + ); + const caseInsensitiveData = 'bugs'; + const qs = createExplainableQuery( + client, + 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' + ); + await adapter + .ensureIndex(tableName, schema, ['objectId'], null, true) .then(() => { - //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's client - .none( - 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)', - [tableName, 'objectId', 'username'] - ) - .then(() => { - const caseInsensitiveData = 'bugs'; - const qs = createExplainableQuery( - client, - 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' + .one(qs, [tableName, 'objectId', caseInsensitiveData]) + .then(explained => { + expect(explained['QUERY PLAN'][0].Plan['Node Type']).not.toContain( + 'Seq Scan' ); - adapter - .ensureIndex( - tableName, - schema, - ['objectId'], - null, - true - ) - .then(() => { - client - .one(qs, [tableName, 'objectId', caseInsensitiveData]) - .then(explained => { - expect( - explained['QUERY PLAN'][0].Plan['Node Type'] - ).not.toContain('Seq Scan'); - expect( - explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] - ).toContain('parse_default'); - //Delete generated data in postgres - dropTable(client, tableName); - }); - }); + expect( + explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] + ).toContain('parse_default'); + //Delete generated data in postgres + return dropTable(client, tableName); }); }); }); From d024fb711a6123fdeba5ec08fbef989fe2fd5647 Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 17:55:14 -0400 Subject: [PATCH 37/50] fixed docker command in Contribute file --- CONTRIBUTING.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 893cf33b0f..5029db9c1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,15 +70,25 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: -```sh -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 20 && docker exec -it parse-postgres psql -U $USER -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +``` +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 20 && docker exec -it parse-postgres psql -U postgres -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: -```sh +``` docker stop parse-postgres ``` +You can also use the [postgis/postgis:11-2.5-alpine](https://hub.docker.com/r/postgis/postgis) image in a Dockerfile and copy this [script](https://github.com/parse-community/parse-server/blob/master/scripts/before_script_postgres.sh) to the image by adding the following lines: + +``` +#Install additional scripts. These are run in abc order during initial start +COPY ./scripts/setup-dbs.sh /docker-entrypoint-initdb.d/setup-dbs.sh +RUN chmod +x /docker-entrypoint-initdb.d/setup-dbs.sh +``` + +Note that the script above will ONLY be executed during initialization of the container with no data in the database, see the official [Postgres image](https://hub.docker.com/_/postgres) for details. If you want to use the script to run again be sure there is no data in the /var/lib/postgresql/data of the container. + ### Generate Parse Server Config Definition If you want to make changes to [Parse Server Configuration][config] add the desired configuration to [src/Options/index.js][config-index] and run `npm run definitions`. This will output [src/Options/Definitions.js][config-def] and [src/Options/docs.js][config-docs]. From 5d34ef23d8e0435bb3eec4503d79b0bf59a58a52 Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 20:54:01 -0400 Subject: [PATCH 38/50] added ability to explain the find method --- package-lock.json | 20 ++++++++++++++ spec/PostgresStorageAdapter.spec.js | 10 ++----- .../Postgres/PostgresStorageAdapter.js | 26 ++++++++++++++----- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/package-lock.json b/package-lock.json index ca98da74d9..3c45bc2880 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,6 +51,7 @@ "dev": true, "requires": { "chokidar": "^2.1.8", + "commander": "^4.0.1", "convert-source-map": "^1.1.0", "fs-readdir-recursive": "^1.1.0", "glob": "^7.0.0", @@ -60,6 +61,12 @@ "source-map": "^0.5.0" }, "dependencies": { + "commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true + }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -8280,6 +8287,7 @@ "dev": true, "requires": { "chalk": "^3.0.0", + "commander": "^4.0.1", "cosmiconfig": "^6.0.0", "debug": "^4.1.1", "dedent": "^0.7.0", @@ -8337,6 +8345,12 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true + }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -10185,6 +10199,7 @@ "@babel/runtime-corejs3": "7.7.7", "crypto-js": "3.1.9-1", "uuid": "3.3.3", + "ws": "7.2.1", "xmlhttprequest": "1.8.0" }, "dependencies": { @@ -10209,6 +10224,11 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.3.tgz", "integrity": "sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ==" + }, + "ws": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz", + "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A==" } } }, diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index 82c7d41cbc..a11e59cfd1 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -12,10 +12,6 @@ const getColumns = (client, className) => { ); }; -const createExplainableQuery = (client, query) => { - return 'EXPLAIN (ANALYZE, FORMAT JSON) ' + query; -}; - const dropTable = (client, className) => { return client.none('DROP TABLE IF EXISTS $', { className }); }; @@ -173,8 +169,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { [tableName, 'objectId', 'username'] ); const caseInsensitiveData = 'bugs'; - const qs = createExplainableQuery( - client, + const qs = adapter.createExplainableQuery( 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' ); await client @@ -224,8 +219,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { [tableName, 'objectId', 'username'] ); const caseInsensitiveData = 'bugs'; - const qs = createExplainableQuery( - client, + const qs = adapter.createExplainableQuery( 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' ); await adapter diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index a88cb225c1..6584750644 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -852,6 +852,10 @@ export class PostgresStorageAdapter implements StorageAdapter { this.canSortOnJoinTables = false; } + createExplainableQuery (query: string) { + return 'EXPLAIN (ANALYZE, FORMAT JSON) ' + query; + } + handleShutdown() { if (!this._client) { return; @@ -1835,7 +1839,7 @@ export class PostgresStorageAdapter implements StorageAdapter { className: string, schema: SchemaType, query: QueryType, - { skip, limit, sort, keys, caseInsensitive }: QueryOptions + { skip, limit, sort, keys, caseInsensitive, explain }: QueryOptions ) { debug('find', className, query, { skip, @@ -1843,6 +1847,7 @@ export class PostgresStorageAdapter implements StorageAdapter { sort, keys, caseInsensitive, + explain, }); const hasLimit = limit !== undefined; const hasSkip = skip !== undefined; @@ -1912,7 +1917,8 @@ export class PostgresStorageAdapter implements StorageAdapter { values = values.concat(keys); } - const qs = `SELECT ${columns} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern}`; + const originaQuery = `SELECT ${columns} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern}`; + const qs = explain ? this.createExplainableQuery(originaQuery) : originaQuery; debug(qs, values); return this._client .any(qs, values) @@ -1923,10 +1929,13 @@ export class PostgresStorageAdapter implements StorageAdapter { } return []; }) - .then(results => - results.map(object => + .then(results => { + if (explain){ + return results; + } + return results.map(object => this.postgresObjectToParseObject(className, object, schema) - ) + );} ); } @@ -2528,14 +2537,17 @@ export class PostgresStorageAdapter implements StorageAdapter { schema: SchemaType, fieldNames: string[], indexName: ?string, - caseInsensitive: boolean = false + caseInsensitive: boolean = false, + conn: ?any = null ): Promise { + + conn = conn != null ? conn : this._client; const defaultIndexName = `parse_default_${fieldNames.sort().join('_')}`; const indexNameOptions: Object = indexName != null ? { name: indexName } : { name: defaultIndexName }; const constraintPatterns = caseInsensitive ? fieldNames.map((fieldName, index) => `lower($${index + 3}:name) varchar_pattern_ops`) : fieldNames.map((fieldName, index) => `$${index + 3}:name`); const qs = `CREATE INDEX $1:name ON $2:name (${constraintPatterns.join()})`; - await this._client.none(qs, [indexNameOptions.name, className, ...fieldNames]) + await conn.none(qs, [indexNameOptions.name, className, ...fieldNames]) .catch(error => { if ( error.code === PostgresDuplicateRelationError && From d148804286aff5a88760fd4173ffad9ea8c85ebc Mon Sep 17 00:00:00 2001 From: Corey Date: Sun, 22 Mar 2020 21:56:09 -0400 Subject: [PATCH 39/50] triggering another build --- spec/PostgresStorageAdapter.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index a11e59cfd1..d96ec034a5 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -234,7 +234,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { expect( explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] ).toContain('parse_default'); - //Delete generated data in postgres + //Delete generated data in postgres by dropping table return dropTable(client, tableName); }); }); From 4da11a0281d7549483e5da16cc478a6cc25f685b Mon Sep 17 00:00:00 2001 From: Corey Date: Mon, 23 Mar 2020 00:01:40 -0400 Subject: [PATCH 40/50] added ability to choose to 'analyze' a query which actually executes (this can be bad when looking at a query plan for Insert, Delete, etc.) the query or to just setup the query plan (default, previous versions defaulted to 'analyze'). Alse added some comparsons on sequential vs index searches for postgres --- spec/PostgresStorageAdapter.spec.js | 54 +++++++++++++++---- .../Postgres/PostgresStorageAdapter.js | 9 +++- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index d96ec034a5..72a5dc12d5 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -169,28 +169,62 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { [tableName, 'objectId', 'username'] ); const caseInsensitiveData = 'bugs'; - const qs = adapter.createExplainableQuery( - 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' - ); + const originalQuery = 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)'; + const analyzedExplainQuery = adapter.createExplainableQuery(originalQuery, true); await client - .one(qs, [tableName, 'objectId', caseInsensitiveData]) + .one(analyzedExplainQuery, [tableName, 'objectId', caseInsensitiveData]) .then(explained => { - expect(explained['QUERY PLAN'][0].Plan['Node Type']).toBe('Seq Scan'); + const preIndexPlan = explained; + + expect(preIndexPlan['QUERY PLAN'][0].Plan['Node Type']).toBe('Seq Scan'); const indexName = 'test_case_insensitive_column'; adapter .ensureIndex(tableName, schema, ['objectId'], indexName, true) .then(() => { client - .one(qs, [tableName, 'objectId', caseInsensitiveData]) + .one(analyzedExplainQuery, [tableName, 'objectId', caseInsensitiveData]) .then(explained => { + const postIndexPlan = explained; + + //Should not be a sequential scan expect( - explained['QUERY PLAN'][0].Plan['Node Type'] + postIndexPlan['QUERY PLAN'][0].Plan['Node Type'] ).not.toContain('Seq Scan'); + + //Should be using the index created for this expect( - explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] + postIndexPlan['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] ).toBe(indexName); - //Delete generated data in postgres - return dropTable(client, tableName); + + //Sequential should take more time to plan than indexed + expect( + preIndexPlan['QUERY PLAN'][0]['Planning Time'] + ).toBeGreaterThan(postIndexPlan['QUERY PLAN'][0]['Planning Time']); + + //Sequential should take more time to execute than indexed + expect( + preIndexPlan['QUERY PLAN'][0]['Execution Time'] + ).toBeGreaterThan(postIndexPlan['QUERY PLAN'][0]['Execution Time']); + + //Test explaining without analyzing + const basicExplainQuery = adapter.createExplainableQuery(originalQuery); + client + .one(basicExplainQuery, [tableName, 'objectId', caseInsensitiveData]) + .then(explained => { + + //Check that basic query plans isn't a sequential scan + expect( + explained['QUERY PLAN'][0].Plan['Node Type'] + ).not.toContain('Seq Scan'); + + //Basic query plans shouldn't have an execution time + expect( + explained['QUERY PLAN'][0]['Execution Time'] + ).toBeUndefined(); + + //Delete generated data in postgres + return dropTable(client, tableName); + }) }); }); }); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 6584750644..cc271ac532 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -852,8 +852,13 @@ export class PostgresStorageAdapter implements StorageAdapter { this.canSortOnJoinTables = false; } - createExplainableQuery (query: string) { - return 'EXPLAIN (ANALYZE, FORMAT JSON) ' + query; + //Note that analyze=true will run the query, executing INSERTS, DELETES, etc. + createExplainableQuery (query: string, analyze:boolean = false) { + if (analyze){ + return 'EXPLAIN (ANALYZE, FORMAT JSON) ' + query; + }else{ + return 'EXPLAIN (FORMAT JSON) ' + query; + } } handleShutdown() { From 54d07bbb4099c59ae937cd198967658c478f0cea Mon Sep 17 00:00:00 2001 From: Corey Date: Mon, 23 Mar 2020 00:16:01 -0400 Subject: [PATCH 41/50] made sure to check that search actually returns 1 result. Removed prep time comparison between searches as this seemed to be variable --- spec/PostgresStorageAdapter.spec.js | 47 ++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index 72a5dc12d5..a51b4177a9 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -169,23 +169,40 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { [tableName, 'objectId', 'username'] ); const caseInsensitiveData = 'bugs'; - const originalQuery = 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)'; - const analyzedExplainQuery = adapter.createExplainableQuery(originalQuery, true); + const originalQuery = + 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)'; + const analyzedExplainQuery = adapter.createExplainableQuery( + originalQuery, + true + ); await client .one(analyzedExplainQuery, [tableName, 'objectId', caseInsensitiveData]) .then(explained => { const preIndexPlan = explained; - expect(preIndexPlan['QUERY PLAN'][0].Plan['Node Type']).toBe('Seq Scan'); + //Make sure search returned with only 1 result + expect(preIndexPlan['QUERY PLAN'][0].Plan['Actual Rows']).toBe(1); + expect(preIndexPlan['QUERY PLAN'][0].Plan['Node Type']).toBe( + 'Seq Scan' + ); const indexName = 'test_case_insensitive_column'; + adapter .ensureIndex(tableName, schema, ['objectId'], indexName, true) .then(() => { client - .one(analyzedExplainQuery, [tableName, 'objectId', caseInsensitiveData]) + .one(analyzedExplainQuery, [ + tableName, + 'objectId', + caseInsensitiveData, + ]) .then(explained => { const postIndexPlan = explained; + //Make sure search returned with only 1 result + expect(postIndexPlan['QUERY PLAN'][0].Plan['Actual Rows']).toBe( + 1 + ); //Should not be a sequential scan expect( postIndexPlan['QUERY PLAN'][0].Plan['Node Type'] @@ -196,22 +213,24 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { postIndexPlan['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] ).toBe(indexName); - //Sequential should take more time to plan than indexed - expect( - preIndexPlan['QUERY PLAN'][0]['Planning Time'] - ).toBeGreaterThan(postIndexPlan['QUERY PLAN'][0]['Planning Time']); - //Sequential should take more time to execute than indexed expect( preIndexPlan['QUERY PLAN'][0]['Execution Time'] - ).toBeGreaterThan(postIndexPlan['QUERY PLAN'][0]['Execution Time']); + ).toBeGreaterThan( + postIndexPlan['QUERY PLAN'][0]['Execution Time'] + ); //Test explaining without analyzing - const basicExplainQuery = adapter.createExplainableQuery(originalQuery); + const basicExplainQuery = adapter.createExplainableQuery( + originalQuery + ); client - .one(basicExplainQuery, [tableName, 'objectId', caseInsensitiveData]) + .one(basicExplainQuery, [ + tableName, + 'objectId', + caseInsensitiveData, + ]) .then(explained => { - //Check that basic query plans isn't a sequential scan expect( explained['QUERY PLAN'][0].Plan['Node Type'] @@ -224,7 +243,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { //Delete generated data in postgres return dropTable(client, tableName); - }) + }); }); }); }); From f60fe059b1241ba70b05ffbdd373145c6c8ae56f Mon Sep 17 00:00:00 2001 From: Corey Date: Mon, 23 Mar 2020 12:17:10 -0400 Subject: [PATCH 42/50] added test cases using find and case insensitivity on fields other than username and password. Also added explain to aggregate method --- spec/PostgresStorageAdapter.spec.js | 123 +++++++++++++----- .../Postgres/PostgresStorageAdapter.js | 64 +++++---- 2 files changed, 128 insertions(+), 59 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index a51b4177a9..ab2608286f 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -246,51 +246,110 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { }); }); }); + }) + .catch(error => { + // Query on non existing table, don't crash + if (error.code !== '42P01') { + throw error; + } + return []; }); }); - it('should use index for caseInsensitive query using default indexname', async () => { - const tableName = 'CaseTable'; - const schema = { - fields: { - objectId: { type: 'String' }, - username: { type: 'String' }, - email: { type: 'String' }, - }, - }; + it('should use index for caseInsensitive query using Parse find', async () => { + const tableName = '_User'; + const user = new Parse.User(); + user.set('username', 'Bugs'); + user.set('password', 'Bunny'); + await user.signUp(); + const database = Config.get(Parse.applicationId).database; + + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's const client = adapter._client; - await dropTable(client, tableName); - await adapter.createTable(tableName, schema); await client.none( - 'INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', - [tableName, 'objectId', 'username', 'Bugs', 'Bunny'] + 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)', + [tableName, 'objectId', 'username'] + ); + const caseInsensitiveData = 'bugs'; + const fieldToSearch = 'username'; + //Check using find method for Parse + const preIndexPlan = await database.find( + tableName, + { username: caseInsensitiveData }, + { caseInsensitive: true, explain: true } + ); + + //Check that basic query plans isn't a sequential scan, be careful as find uses "any" to query + expect( + preIndexPlan[0]['QUERY PLAN'][0].Plan['Node Type']).toBe('Seq Scan' + ); + + //Basic query plans shouldn't have an execution time + expect( + preIndexPlan[0]['QUERY PLAN'][0]['Execution Time'] + ).toBeUndefined(); + + const indexName = 'test_case_insensitive_column'; + const schema = await new Parse.Schema('_User').get(); + await adapter + .ensureIndex(tableName, schema, [fieldToSearch], indexName, true); + + //Check using find method for Parse + const postIndexPlan = await database.find( + tableName, + { username: caseInsensitiveData }, + { caseInsensitive: true, explain: true } ); + //Check that basic query plans isn't a sequential scan + expect( + postIndexPlan[0]['QUERY PLAN'][0].Plan['Node Type'] + ).not.toContain('Seq Scan'); + + //Basic query plans shouldn't have an execution time + expect( + postIndexPlan[0]['QUERY PLAN'][0]['Execution Time'] + ).toBeUndefined(); + //Delete generated data in postgres by dropping table + return dropTable(client, tableName); + }); + + it('should use index for caseInsensitive query using default indexname and non username/email field', async () => { + const tableName = '_User'; + const user = new Parse.User(); + user.set('username', 'Bugs'); + user.set('password', 'Bunny'); + await user.signUp(); + const database = Config.get(Parse.applicationId).database; + const fieldToSearch = 'objectId'; + //Create index before data is inserted + const schema = await new Parse.Schema('_User').get(); + await adapter + .ensureIndex(tableName, schema, [fieldToSearch], null, true); + //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's + const client = adapter._client; await client.none( 'INSERT INTO $1:name ($2:name, $3:name) SELECT MD5(random()::text), MD5(random()::text) FROM generate_series(1,5000)', [tableName, 'objectId', 'username'] ); - const caseInsensitiveData = 'bugs'; - const qs = adapter.createExplainableQuery( - 'SELECT * FROM $1:name WHERE lower($2:name)=lower($3)' + + const caseInsensitiveData = 'bunny'; + //Check using find method for Parse + const indexPlan = await database.find( + tableName, + { objectId: caseInsensitiveData }, + { caseInsensitive: true, explain: true } ); - await adapter - .ensureIndex(tableName, schema, ['objectId'], null, true) - .then(() => { - client - .one(qs, [tableName, 'objectId', caseInsensitiveData]) - .then(explained => { - expect(explained['QUERY PLAN'][0].Plan['Node Type']).not.toContain( - 'Seq Scan' - ); - expect( - explained['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] - ).toContain('parse_default'); - //Delete generated data in postgres by dropping table - return dropTable(client, tableName); - }); - }); + + expect(indexPlan[0]['QUERY PLAN'][0].Plan['Node Type']).not.toContain( + 'Seq Scan' + ); + expect( + indexPlan[0]['QUERY PLAN'][0].Plan['Index Name'] + ).toContain('parse_default'); + //Delete generated data in postgres by dropping table + return dropTable(client, tableName); }); }); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index cc271ac532..d1b5facc86 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -286,8 +286,7 @@ const buildWhereClause = ({ // TODO: Handle querying by _auth_data_provider, authData is stored in authData field continue; } else if ( - caseInsensitive && - (fieldName === 'username' || fieldName === 'email') + caseInsensitive ) { patterns.push(`LOWER($${index}:name) = LOWER($${index + 1})`); values.push(fieldName, fieldValue); @@ -1922,8 +1921,8 @@ export class PostgresStorageAdapter implements StorageAdapter { values = values.concat(keys); } - const originaQuery = `SELECT ${columns} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern}`; - const qs = explain ? this.createExplainableQuery(originaQuery) : originaQuery; + const originalQuery = `SELECT ${columns} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern}`; + const qs = explain ? this.createExplainableQuery(originalQuery) : originalQuery; debug(qs, values); return this._client .any(qs, values) @@ -2198,7 +2197,13 @@ export class PostgresStorageAdapter implements StorageAdapter { ); } - async aggregate(className: string, schema: any, pipeline: any) { + async aggregate( + className: string, + schema: any, + pipeline: any, + readPreference: ?string, + hint: ?mixed, + explain?: boolean) { debug('aggregate', className, pipeline); const values = [className]; let index: number = 2; @@ -2383,31 +2388,36 @@ export class PostgresStorageAdapter implements StorageAdapter { sort !== undefined && sorting.length > 0 ? `ORDER BY ${sorting}` : ''; } } - - const qs = `SELECT ${columns.join()} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern} ${groupPattern}`; + const originalQuery = `SELECT ${columns.join()} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern} ${groupPattern}`; + const qs = explain ? this.createExplainableQuery(originalQuery) : originalQuery; debug(qs, values); return this._client - .map(qs, values, a => - this.postgresObjectToParseObject(className, a, schema) - ) .then(results => { - results.forEach(result => { - if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { - result.objectId = null; - } - if (groupValues) { - result.objectId = {}; - for (const key in groupValues) { - result.objectId[key] = result[key]; - delete result[key]; - } - } - if (countField) { - result[countField] = parseInt(result[countField], 10); - } - }); - return results; - }); + if (explain){ + return results; + } + results.map(qs, values, a => + this.postgresObjectToParseObject(className, a, schema) + ) + .then(results => { + results.forEach(result => { + if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { + result.objectId = null; + } + if (groupValues) { + result.objectId = {}; + for (const key in groupValues) { + result.objectId[key] = result[key]; + delete result[key]; + } + } + if (countField) { + result[countField] = parseInt(result[countField], 10); + } + }); + return results; + }); + }) } async performInitialization({ VolatileClassesSchemas }: any) { From f1f9d87e6f19561f95bf36764274858f06eb02f0 Mon Sep 17 00:00:00 2001 From: Corey Date: Mon, 23 Mar 2020 12:57:02 -0400 Subject: [PATCH 43/50] fixing issue where query in aggregate replaced the map method incorrectly --- src/Adapters/Storage/Postgres/PostgresStorageAdapter.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index d1b5facc86..34dcb4551a 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2204,7 +2204,7 @@ export class PostgresStorageAdapter implements StorageAdapter { readPreference: ?string, hint: ?mixed, explain?: boolean) { - debug('aggregate', className, pipeline); + debug('aggregate', className, pipeline, readPreference, hint, explain); const values = [className]; let index: number = 2; let columns: string[] = []; @@ -2392,12 +2392,13 @@ export class PostgresStorageAdapter implements StorageAdapter { const qs = explain ? this.createExplainableQuery(originalQuery) : originalQuery; debug(qs, values); return this._client + .any(qs, values) .then(results => { if (explain){ return results; } - results.map(qs, values, a => - this.postgresObjectToParseObject(className, a, schema) + results.map( + this.postgresObjectToParseObject(className, results, schema) ) .then(results => { results.forEach(result => { @@ -2417,7 +2418,7 @@ export class PostgresStorageAdapter implements StorageAdapter { }); return results; }); - }) + }); } async performInitialization({ VolatileClassesSchemas }: any) { From 21df1e2704646cc154fa806f2dd86b1af55721ca Mon Sep 17 00:00:00 2001 From: Corey Date: Mon, 23 Mar 2020 13:31:07 -0400 Subject: [PATCH 44/50] reverted back to mapping for aggregate method to make sure it's the issue --- .../Postgres/PostgresStorageAdapter.js | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 34dcb4551a..5503f3546a 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2392,33 +2392,27 @@ export class PostgresStorageAdapter implements StorageAdapter { const qs = explain ? this.createExplainableQuery(originalQuery) : originalQuery; debug(qs, values); return this._client - .any(qs, values) - .then(results => { - if (explain){ - return results; + .map(qs, values, a => + this.postgresObjectToParseObject(className, a, schema) + ) + .then(results => { + results.forEach(result => { + if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { + result.objectId = null; + } + if (groupValues) { + result.objectId = {}; + for (const key in groupValues) { + result.objectId[key] = result[key]; + delete result[key]; + } + } + if (countField) { + result[countField] = parseInt(result[countField], 10); } - results.map( - this.postgresObjectToParseObject(className, results, schema) - ) - .then(results => { - results.forEach(result => { - if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { - result.objectId = null; - } - if (groupValues) { - result.objectId = {}; - for (const key in groupValues) { - result.objectId[key] = result[key]; - delete result[key]; - } - } - if (countField) { - result[countField] = parseInt(result[countField], 10); - } - }); - return results; - }); }); + return results; + }); } async performInitialization({ VolatileClassesSchemas }: any) { From d8576c981de032d07d9a6edcf1347251b5fd2fe6 Mon Sep 17 00:00:00 2001 From: Corey Date: Mon, 23 Mar 2020 15:33:06 -0400 Subject: [PATCH 45/50] switched back to caseInsensitive check for email and username as it was causing issues --- spec/PostgresStorageAdapter.spec.js | 8 +-- .../Postgres/PostgresStorageAdapter.js | 67 +++++++++++++------ 2 files changed, 52 insertions(+), 23 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index ab2608286f..78508ea3b9 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -314,14 +314,14 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { return dropTable(client, tableName); }); - it('should use index for caseInsensitive query using default indexname and non username/email field', async () => { + it('should use index for caseInsensitive query using default indexname', async () => { const tableName = '_User'; const user = new Parse.User(); user.set('username', 'Bugs'); user.set('password', 'Bunny'); await user.signUp(); const database = Config.get(Parse.applicationId).database; - const fieldToSearch = 'objectId'; + const fieldToSearch = 'username'; //Create index before data is inserted const schema = await new Parse.Schema('_User').get(); await adapter @@ -334,11 +334,11 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { [tableName, 'objectId', 'username'] ); - const caseInsensitiveData = 'bunny'; + const caseInsensitiveData = 'buGs'; //Check using find method for Parse const indexPlan = await database.find( tableName, - { objectId: caseInsensitiveData }, + { username: caseInsensitiveData }, { caseInsensitive: true, explain: true } ); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 5503f3546a..2b403db026 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -286,7 +286,8 @@ const buildWhereClause = ({ // TODO: Handle querying by _auth_data_provider, authData is stored in authData field continue; } else if ( - caseInsensitive + caseInsensitive && + (fieldName === 'username' || fieldName === 'email') ) { patterns.push(`LOWER($${index}:name) = LOWER($${index + 1})`); values.push(fieldName, fieldValue); @@ -2392,27 +2393,55 @@ export class PostgresStorageAdapter implements StorageAdapter { const qs = explain ? this.createExplainableQuery(originalQuery) : originalQuery; debug(qs, values); return this._client - .map(qs, values, a => - this.postgresObjectToParseObject(className, a, schema) - ) - .then(results => { - results.forEach(result => { - if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { - result.objectId = null; + /*.any(qs, values) + .then(a => { + if (explain){ + return results; } - if (groupValues) { - result.objectId = {}; - for (const key in groupValues) { - result.objectId[key] = result[key]; - delete result[key]; + //There's a bug here + a.map(object => { + return this.postgresObjectToParseObject(className, object, schema) + }) + .then(results => { + results.forEach(result => { + if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { + result.objectId = null; + } + if (groupValues) { + result.objectId = {}; + for (const key in groupValues) { + result.objectId[key] = result[key]; + delete result[key]; + } + } + if (countField) { + result[countField] = parseInt(result[countField], 10); + } + }); + return results; + }); + });*/ + .map(qs, values, a => + this.postgresObjectToParseObject(className, a, schema) + ) + .then(results => { + results.forEach(result => { + if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { + result.objectId = null; } - } - if (countField) { - result[countField] = parseInt(result[countField], 10); - } + if (groupValues) { + result.objectId = {}; + for (const key in groupValues) { + result.objectId[key] = result[key]; + delete result[key]; + } + } + if (countField) { + result[countField] = parseInt(result[countField], 10); + } + }); + return results; }); - return results; - }); } async performInitialization({ VolatileClassesSchemas }: any) { From 6042f23ee7ff1704957946fd3ee9bab8fc02bbb6 Mon Sep 17 00:00:00 2001 From: Corey Date: Mon, 23 Mar 2020 20:05:49 -0400 Subject: [PATCH 46/50] fixed aggregate method using explain --- .../Postgres/PostgresStorageAdapter.js | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 2b403db026..cac157fa7b 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -2393,38 +2393,14 @@ export class PostgresStorageAdapter implements StorageAdapter { const qs = explain ? this.createExplainableQuery(originalQuery) : originalQuery; debug(qs, values); return this._client - /*.any(qs, values) + .any(qs, values) .then(a => { if (explain){ - return results; + return a; } - //There's a bug here - a.map(object => { - return this.postgresObjectToParseObject(className, object, schema) - }) - .then(results => { - results.forEach(result => { - if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { - result.objectId = null; - } - if (groupValues) { - result.objectId = {}; - for (const key in groupValues) { - result.objectId[key] = result[key]; - delete result[key]; - } - } - if (countField) { - result[countField] = parseInt(result[countField], 10); - } - }); - return results; - }); - });*/ - .map(qs, values, a => - this.postgresObjectToParseObject(className, a, schema) - ) - .then(results => { + const results = a.map(object => + this.postgresObjectToParseObject(className, object, schema) + ); results.forEach(result => { if (!Object.prototype.hasOwnProperty.call(result, 'objectId')) { result.objectId = null; From 93a17dde419305fbfded3991de23c115c775a750 Mon Sep 17 00:00:00 2001 From: Corey Date: Tue, 24 Mar 2020 10:17:29 -0400 Subject: [PATCH 47/50] made query plain results more flexible/reusable. Got rid of droptables as 'beforeEach' already handles this --- spec/PostgresStorageAdapter.spec.js | 144 +++++++++++++--------------- 1 file changed, 69 insertions(+), 75 deletions(-) diff --git a/spec/PostgresStorageAdapter.spec.js b/spec/PostgresStorageAdapter.spec.js index 78508ea3b9..e8777b4901 100644 --- a/spec/PostgresStorageAdapter.spec.js +++ b/spec/PostgresStorageAdapter.spec.js @@ -85,7 +85,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { expect(columns).toContain('columnA'); expect(columns).toContain('columnB'); expect(columns).toContain('columnC'); - dropTable(client, className); + done(); }) .catch(error => done.fail(error)); @@ -114,7 +114,6 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { expect(columns.length).toEqual(2); expect(columns).toContain('columnA'); expect(columns).toContain('columnB'); - dropTable(client, className); done(); }) .catch(done); @@ -147,7 +146,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { ); }); - it('should use index for caseInsensitive query', async () => { + it('should use index for caseInsensitive query using Postgres', async () => { const tableName = '_User'; const schema = { fields: { @@ -157,7 +156,6 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { }, }; const client = adapter._client; - await dropTable(client, tableName); await adapter.createTable(tableName, schema); await client.none( 'INSERT INTO $1:name ($2:name, $3:name) VALUES ($4, $5)', @@ -180,11 +178,11 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { .then(explained => { const preIndexPlan = explained; - //Make sure search returned with only 1 result - expect(preIndexPlan['QUERY PLAN'][0].Plan['Actual Rows']).toBe(1); - expect(preIndexPlan['QUERY PLAN'][0].Plan['Node Type']).toBe( - 'Seq Scan' - ); + preIndexPlan['QUERY PLAN'].forEach(element => { + //Make sure search returned with only 1 result + expect(element.Plan['Actual Rows']).toBe(1); + expect(element.Plan['Node Type']).toBe('Seq Scan'); + }); const indexName = 'test_case_insensitive_column'; adapter @@ -199,26 +197,27 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { .then(explained => { const postIndexPlan = explained; - //Make sure search returned with only 1 result - expect(postIndexPlan['QUERY PLAN'][0].Plan['Actual Rows']).toBe( - 1 - ); - //Should not be a sequential scan - expect( - postIndexPlan['QUERY PLAN'][0].Plan['Node Type'] - ).not.toContain('Seq Scan'); - - //Should be using the index created for this - expect( - postIndexPlan['QUERY PLAN'][0].Plan.Plans[0]['Index Name'] - ).toBe(indexName); - - //Sequential should take more time to execute than indexed - expect( - preIndexPlan['QUERY PLAN'][0]['Execution Time'] - ).toBeGreaterThan( - postIndexPlan['QUERY PLAN'][0]['Execution Time'] - ); + postIndexPlan['QUERY PLAN'].forEach(element => { + //Make sure search returned with only 1 result + expect(element.Plan['Actual Rows']).toBe(1); + //Should not be a sequential scan + expect(element.Plan['Node Type']).not.toContain('Seq Scan'); + + //Should be using the index created for this + element.Plan.Plans.forEach(innerElement => { + expect(innerElement['Index Name']).toBe(indexName); + }); + }); + + //These are the same query so should be the same size + for (let i = 0; i < preIndexPlan['QUERY PLAN'].length; i++) { + //Sequential should take more time to execute than indexed + expect( + preIndexPlan['QUERY PLAN'][i]['Execution Time'] + ).toBeGreaterThan( + postIndexPlan['QUERY PLAN'][i]['Execution Time'] + ); + } //Test explaining without analyzing const basicExplainQuery = adapter.createExplainableQuery( @@ -231,18 +230,15 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { caseInsensitiveData, ]) .then(explained => { - //Check that basic query plans isn't a sequential scan - expect( - explained['QUERY PLAN'][0].Plan['Node Type'] - ).not.toContain('Seq Scan'); - - //Basic query plans shouldn't have an execution time - expect( - explained['QUERY PLAN'][0]['Execution Time'] - ).toBeUndefined(); - - //Delete generated data in postgres - return dropTable(client, tableName); + explained['QUERY PLAN'].forEach(element => { + //Check that basic query plans isn't a sequential scan + expect(element.Plan['Node Type']).not.toContain( + 'Seq Scan' + ); + + //Basic query plans shouldn't have an execution time + expect(element['Execution Time']).toBeUndefined(); + }); }); }); }); @@ -256,7 +252,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { }); }); - it('should use index for caseInsensitive query using Parse find', async () => { + it('should use index for caseInsensitive query', async () => { const tableName = '_User'; const user = new Parse.User(); user.set('username', 'Bugs'); @@ -279,20 +275,24 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { { caseInsensitive: true, explain: true } ); - //Check that basic query plans isn't a sequential scan, be careful as find uses "any" to query - expect( - preIndexPlan[0]['QUERY PLAN'][0].Plan['Node Type']).toBe('Seq Scan' - ); - - //Basic query plans shouldn't have an execution time - expect( - preIndexPlan[0]['QUERY PLAN'][0]['Execution Time'] - ).toBeUndefined(); + preIndexPlan.forEach(element => { + element['QUERY PLAN'].forEach(innerElement => { + //Check that basic query plans isn't a sequential scan, be careful as find uses "any" to query + expect(innerElement.Plan['Node Type']).toBe('Seq Scan'); + //Basic query plans shouldn't have an execution time + expect(innerElement['Execution Time']).toBeUndefined(); + }); + }); const indexName = 'test_case_insensitive_column'; const schema = await new Parse.Schema('_User').get(); - await adapter - .ensureIndex(tableName, schema, [fieldToSearch], indexName, true); + await adapter.ensureIndex( + tableName, + schema, + [fieldToSearch], + indexName, + true + ); //Check using find method for Parse const postIndexPlan = await database.find( @@ -301,17 +301,15 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { { caseInsensitive: true, explain: true } ); - //Check that basic query plans isn't a sequential scan - expect( - postIndexPlan[0]['QUERY PLAN'][0].Plan['Node Type'] - ).not.toContain('Seq Scan'); - - //Basic query plans shouldn't have an execution time - expect( - postIndexPlan[0]['QUERY PLAN'][0]['Execution Time'] - ).toBeUndefined(); - //Delete generated data in postgres by dropping table - return dropTable(client, tableName); + postIndexPlan.forEach(element => { + element['QUERY PLAN'].forEach(innerElement => { + //Check that basic query plans isn't a sequential scan + expect(innerElement.Plan['Node Type']).not.toContain('Seq Scan'); + + //Basic query plans shouldn't have an execution time + expect(innerElement['Execution Time']).toBeUndefined(); + }); + }); }); it('should use index for caseInsensitive query using default indexname', async () => { @@ -324,8 +322,7 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { const fieldToSearch = 'username'; //Create index before data is inserted const schema = await new Parse.Schema('_User').get(); - await adapter - .ensureIndex(tableName, schema, [fieldToSearch], null, true); + await adapter.ensureIndex(tableName, schema, [fieldToSearch], null, true); //Postgres won't take advantage of the index until it has a lot of records because sequential is faster for small db's const client = adapter._client; @@ -341,15 +338,12 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => { { username: caseInsensitiveData }, { caseInsensitive: true, explain: true } ); - - expect(indexPlan[0]['QUERY PLAN'][0].Plan['Node Type']).not.toContain( - 'Seq Scan' - ); - expect( - indexPlan[0]['QUERY PLAN'][0].Plan['Index Name'] - ).toContain('parse_default'); - //Delete generated data in postgres by dropping table - return dropTable(client, tableName); + indexPlan.forEach(element => { + element['QUERY PLAN'].forEach(innerElement => { + expect(innerElement.Plan['Node Type']).not.toContain('Seq Scan'); + expect(innerElement.Plan['Index Name']).toContain('parse_default'); + }); + }); }); }); From 3a9f762c650be3a9112d7c23481fde3773917965 Mon Sep 17 00:00:00 2001 From: Corey Date: Tue, 24 Mar 2020 13:19:40 -0400 Subject: [PATCH 48/50] updated CONTRIBUTING doc to use netrecon as default username for postgres (similar to old style). Note that the official postgres docker image for postgres requires POSTGRES_PASSWORD to be set in order to use the image --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5029db9c1f..a0385be444 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ If your pull request introduces a change that may affect the storage or retrieva * Run the tests against the postgres database with `PARSE_SERVER_TEST_DB=postgres npm test`. You'll need to have postgres running on your machine and your configuratoin file setup [appropriately](https://github.com/postgres/postgres/blob/master/src/backend/utils/misc/postgresql.conf.sample#L63) along with [PostGIS](https://postgis.net) 2.2.0 higher installed, or use [`Docker`](#run-a-parse-postgres-with-docker). * The Postgres adapter has a special debugger that traces all the sql commands. You can enable it with setting the environment variable `PARSE_SERVER_LOG_LEVEL=debug` * If your feature is intended to only work with MongoDB, you should disable PostgreSQL-specific tests with: - + - `describe_only_db('mongo')` // will create a `describe` that runs only on mongoDB - `it_only_db('mongo')` // will make a test that only runs on mongo - `it_exclude_dbs(['postgres'])` // will make a test that runs against all DB's but postgres @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ``` -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 20 && docker exec -it parse-postgres psql -U postgres -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-3.0-alpine && sleep 20 && docker exec -it parse-postgres psql -U $USER -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: @@ -79,7 +79,7 @@ To stop the Postgres instance: docker stop parse-postgres ``` -You can also use the [postgis/postgis:11-2.5-alpine](https://hub.docker.com/r/postgis/postgis) image in a Dockerfile and copy this [script](https://github.com/parse-community/parse-server/blob/master/scripts/before_script_postgres.sh) to the image by adding the following lines: +You can also use the [postgis/postgis:11-3.0-alpine](https://hub.docker.com/r/postgis/postgis) image in a Dockerfile and copy this [script](https://github.com/parse-community/parse-server/blob/master/scripts/before_script_postgres.sh) to the image by adding the following lines: ``` #Install additional scripts. These are run in abc order during initial start @@ -91,7 +91,7 @@ Note that the script above will ONLY be executed during initialization of the co ### Generate Parse Server Config Definition -If you want to make changes to [Parse Server Configuration][config] add the desired configuration to [src/Options/index.js][config-index] and run `npm run definitions`. This will output [src/Options/Definitions.js][config-def] and [src/Options/docs.js][config-docs]. +If you want to make changes to [Parse Server Configuration][config] add the desired configuration to [src/Options/index.js][config-index] and run `npm run definitions`. This will output [src/Options/Definitions.js][config-def] and [src/Options/docs.js][config-docs]. To view docs run `npm run docs` and check the `/out` directory. From e4db402565af0d816c56c0f3d69a127f3ce046ed Mon Sep 17 00:00:00 2001 From: Corey Date: Tue, 24 Mar 2020 13:23:18 -0400 Subject: [PATCH 49/50] left postgis at 2.5 in the contributing document as this is the last version to be backwards compatibile with older versions of parse server --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0385be444..b826d7d795 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ``` -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-3.0-alpine && sleep 20 && docker exec -it parse-postgres psql -U $USER -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 20 && docker exec -it parse-postgres psql -U $USER -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: @@ -79,7 +79,7 @@ To stop the Postgres instance: docker stop parse-postgres ``` -You can also use the [postgis/postgis:11-3.0-alpine](https://hub.docker.com/r/postgis/postgis) image in a Dockerfile and copy this [script](https://github.com/parse-community/parse-server/blob/master/scripts/before_script_postgres.sh) to the image by adding the following lines: +You can also use the [postgis/postgis:11-2.5-alpine](https://hub.docker.com/r/postgis/postgis) image in a Dockerfile and copy this [script](https://github.com/parse-community/parse-server/blob/master/scripts/before_script_postgres.sh) to the image by adding the following lines: ``` #Install additional scripts. These are run in abc order during initial start From db936a2019ded6c7b599fa8dbc3afbd6ece0b2ff Mon Sep 17 00:00:00 2001 From: Corey Date: Thu, 26 Mar 2020 08:45:01 -0400 Subject: [PATCH 50/50] updating docker command for postgres --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b826d7d795..1dcf05ba25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ If your pull request introduces a change that may affect the storage or retrieva [PostGIS images (select one with v2.2 or higher) on docker dashboard](https://hub.docker.com/r/postgis/postgis) is based off of the official [postgres](https://registry.hub.docker.com/_/postgres/) image and will work out-of-the-box (as long as you create a user with the necessary extensions for each of your Parse databases; see below). To launch the compatible Postgres instance, copy and paste the following line into your shell: ``` -docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_USER=$USER -e POSTGRES_PASSWORD=postgres --rm postgis/postgis:11-2.5-alpine && sleep 20 && docker exec -it parse-postgres psql -U $USER -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U $USER -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database +docker run -d --name parse-postgres -p 5432:5432 -e POSTGRES_PASSWORD=password --rm postgis/postgis:11-3.0-alpine && sleep 20 && docker exec -it parse-postgres psql -U postgres -c 'CREATE DATABASE parse_server_postgres_adapter_test_database;' && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis;' -d parse_server_postgres_adapter_test_database && docker exec -it parse-postgres psql -U postgres -c 'CREATE EXTENSION postgis_topology;' -d parse_server_postgres_adapter_test_database ``` To stop the Postgres instance: