feat: Add option schemaCacheTtl for schema cache pulling as alternative to enableSchemaHooks (#8436)

Author: dblythy

spec/SchemaPerformance.spec.js

@@ -204,4 +204,58 @@ describe('Schema Performance', function () {
     );
     expect(getAllSpy.calls.count()).toBe(2);
   });
+
+  it('does reload with schemaCacheTtl', async () => {
+    const databaseURI =
+      process.env.PARSE_SERVER_TEST_DB === 'postgres'
+        ? process.env.PARSE_SERVER_TEST_DATABASE_URI
+        : 'mongodb://localhost:27017/parseServerMongoAdapterTestDatabase';
+    await reconfigureServer({
+      databaseAdapter: undefined,
+      databaseURI,
+      silent: false,
+      databaseOptions: { schemaCacheTtl: 1000 },
+    });
+    const SchemaController = require('../lib/Controllers/SchemaController').SchemaController;
+    const spy = spyOn(SchemaController.prototype, 'reloadData').and.callThrough();
+    Object.defineProperty(spy, 'reloadCalls', {
+      get: () => spy.calls.all().filter(call => call.args[0].clearCache).length,
+    });
+
+    const object = new TestObject();
+    object.set('foo', 'bar');
+    await object.save();
+
+    spy.calls.reset();
+
+    object.set('foo', 'bar');
+    await object.save();
+
+    expect(spy.reloadCalls).toBe(0);
+
+    await new Promise(resolve => setTimeout(resolve, 1100));
+
+    object.set('foo', 'bar');
+    await object.save();
+
+    expect(spy.reloadCalls).toBe(1);
+  });
+
+  it('cannot set invalid databaseOptions', async () => {
+    const expectError = async (key, value, expected) =>
+      expectAsync(
+        reconfigureServer({ databaseAdapter: undefined, databaseOptions: { [key]: value } })
+      ).toBeRejectedWith(`databaseOptions.${key} must be a ${expected}`);
+    for (const databaseOptions of [[], 0, 'string']) {
+      await expectAsync(
+        reconfigureServer({ databaseAdapter: undefined, databaseOptions })
+      ).toBeRejectedWith(`databaseOptions must be an object`);
+    }
+    for (const value of [null, 0, 'string', {}, []]) {
+      await expectError('enableSchemaHooks', value, 'boolean');
+    }
+    for (const value of [null, false, 'string', {}, []]) {
+      await expectError('schemaCacheTtl', value, 'number');
+    }
+  });
 });

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -139,11 +139,12 @@ export class MongoStorageAdapter implements StorageAdapter {
   _maxTimeMS: ?number;
   canSortOnJoinTables: boolean;
   enableSchemaHooks: boolean;
+  schemaCacheTtl: ?number;
 
   constructor({ uri = defaults.DefaultMongoURI, collectionPrefix = '', mongoOptions = {} }: any) {
     this._uri = uri;
     this._collectionPrefix = collectionPrefix;
-    this._mongoOptions = mongoOptions;
+    this._mongoOptions = { ...mongoOptions };
     this._mongoOptions.useNewUrlParser = true;
     this._mongoOptions.useUnifiedTopology = true;
     this._onchange = () => {};
@@ -152,8 +153,11 @@ export class MongoStorageAdapter implements StorageAdapter {
     this._maxTimeMS = mongoOptions.maxTimeMS;
     this.canSortOnJoinTables = true;
     this.enableSchemaHooks = !!mongoOptions.enableSchemaHooks;
-    delete mongoOptions.enableSchemaHooks;
-    delete mongoOptions.maxTimeMS;
+    this.schemaCacheTtl = mongoOptions.schemaCacheTtl;
+    for (const key of ['enableSchemaHooks', 'schemaCacheTtl', 'maxTimeMS']) {
+      delete mongoOptions[key];
+      delete this._mongoOptions[key];
+    }
   }
 
   watch(callback: () => void): void {

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -850,13 +850,18 @@ export class PostgresStorageAdapter implements StorageAdapter {
   _pgp: any;
   _stream: any;
   _uuid: any;
+  schemaCacheTtl: ?number;
 
   constructor({ uri, collectionPrefix = '', databaseOptions = {} }: any) {
+    const options = { ...databaseOptions };
     this._collectionPrefix = collectionPrefix;
     this.enableSchemaHooks = !!databaseOptions.enableSchemaHooks;
-    delete databaseOptions.enableSchemaHooks;
+    this.schemaCacheTtl = databaseOptions.schemaCacheTtl;
+    for (const key of ['enableSchemaHooks', 'schemaCacheTtl']) {
+      delete options[key];
+    }
 
-    const { client, pgp } = createClient(uri, databaseOptions);
+    const { client, pgp } = createClient(uri, options);
     this._client = client;
     this._onchange = () => {};
     this._pgp = pgp;

src/Adapters/Storage/StorageAdapter.js

@@ -30,6 +30,8 @@ export type FullQueryOptions = QueryOptions & UpdateQueryOptions;
 
 export interface StorageAdapter {
   canSortOnJoinTables: boolean;
+  schemaCacheTtl: ?number;
+  enableSchemaHooks: boolean;
 
   classExists(className: string): Promise<boolean>;
   setClassLevelPermissions(className: string, clps: any): Promise<void>;

src/Config.js

@@ -9,6 +9,7 @@ import DatabaseController from './Controllers/DatabaseController';
 import { logLevels as validLogLevels } from './Controllers/LoggerController';
 import {
   AccountLockoutOptions,
+  DatabaseOptions,
   FileUploadOptions,
   IdempotencyOptions,
   LogLevels,
@@ -52,23 +53,20 @@ export class Config {
   }
 
   static put(serverConfiguration) {
-    Config.validate(serverConfiguration);
+    Config.validateOptions(serverConfiguration);
+    Config.validateControllers(serverConfiguration);
     AppCache.put(serverConfiguration.appId, serverConfiguration);
     Config.setupPasswordValidator(serverConfiguration.passwordPolicy);
     return serverConfiguration;
   }
 
-  static validate({
-    verifyUserEmails,
-    userController,
-    appName,
+  static validateOptions({
     publicServerURL,
     revokeSessionOnPasswordReset,
     expireInactiveSessions,
     sessionLength,
     defaultLimit,
     maxLimit,
-    emailVerifyTokenValidityDuration,
     accountLockout,
     passwordPolicy,
     masterKeyIps,
@@ -78,7 +76,6 @@ export class Config {
     readOnlyMasterKey,
     allowHeaders,
     idempotencyOptions,
-    emailVerifyTokenReuseIfValid,
     fileUpload,
     pages,
     security,
@@ -88,6 +85,7 @@ export class Config {
     allowExpiredAuthDataToken,
     logLevels,
     rateLimit,
+    databaseOptions,
   }) {
     if (masterKey === readOnlyMasterKey) {
       throw new Error('masterKey and readOnlyMasterKey should be different');
@@ -97,17 +95,6 @@ export class Config {
       throw new Error('masterKey and maintenanceKey should be different');
     }
 
-    const emailAdapter = userController.adapter;
-    if (verifyUserEmails) {
-      this.validateEmailConfiguration({
-        emailAdapter,
-        appName,
-        publicServerURL,
-        emailVerifyTokenValidityDuration,
-        emailVerifyTokenReuseIfValid,
-      });
-    }
-
     this.validateAccountLockoutPolicy(accountLockout);
     this.validatePasswordPolicy(passwordPolicy);
     this.validateFileUploadOptions(fileUpload);
@@ -136,6 +123,27 @@ export class Config {
     this.validateRequestKeywordDenylist(requestKeywordDenylist);
     this.validateRateLimit(rateLimit);
     this.validateLogLevels(logLevels);
+    this.validateDatabaseOptions(databaseOptions);
+  }
+
+  static validateControllers({
+    verifyUserEmails,
+    userController,
+    appName,
+    publicServerURL,
+    emailVerifyTokenValidityDuration,
+    emailVerifyTokenReuseIfValid,
+  }) {
+    const emailAdapter = userController.adapter;
+    if (verifyUserEmails) {
+      this.validateEmailConfiguration({
+        emailAdapter,
+        appName,
+        publicServerURL,
+        emailVerifyTokenValidityDuration,
+        emailVerifyTokenReuseIfValid,
+      });
+    }
   }
 
   static validateRequestKeywordDenylist(requestKeywordDenylist) {
@@ -533,6 +541,25 @@ export class Config {
     }
   }
 
+  static validateDatabaseOptions(databaseOptions) {
+    if (databaseOptions == undefined) {
+      return;
+    }
+    if (Object.prototype.toString.call(databaseOptions) !== '[object Object]') {
+      throw `databaseOptions must be an object`;
+    }
+    if (databaseOptions.enableSchemaHooks === undefined) {
+      databaseOptions.enableSchemaHooks = DatabaseOptions.enableSchemaHooks.default;
+    } else if (typeof databaseOptions.enableSchemaHooks !== 'boolean') {
+      throw `databaseOptions.enableSchemaHooks must be a boolean`;
+    }
+    if (databaseOptions.schemaCacheTtl === undefined) {
+      databaseOptions.schemaCacheTtl = DatabaseOptions.schemaCacheTtl.default;
+    } else if (typeof databaseOptions.schemaCacheTtl !== 'number') {
+      throw `databaseOptions.schemaCacheTtl must be a number`;
+    }
+  }
+
   static validateRateLimit(rateLimit) {
     if (!rateLimit) {
       return;

src/Controllers/SchemaController.js

@@ -682,6 +682,10 @@ const typeToString = (type: SchemaField | string): string => {
   }
   return `${type.type}`;
 };
+const ttl = {
+  date: Date.now(),
+  duration: undefined,
+};
 
 // Stores the entire schema of the app in a weird hybrid format somewhere between
 // the mongo format and the Parse format. Soon, this will all be Parse format.
@@ -694,10 +698,11 @@ export default class SchemaController {
 
   constructor(databaseAdapter: StorageAdapter) {
     this._dbAdapter = databaseAdapter;
+    const config = Config.get(Parse.applicationId);
     this.schemaData = new SchemaData(SchemaCache.all(), this.protectedFields);
-    this.protectedFields = Config.get(Parse.applicationId).protectedFields;
+    this.protectedFields = config.protectedFields;
 
-    const customIds = Config.get(Parse.applicationId).allowCustomObjectId;
+    const customIds = config.allowCustomObjectId;
 
     const customIdRegEx = /^.{1,}$/u; // 1+ chars
     const autoIdRegEx = /^[a-zA-Z0-9]{1,}$/;
@@ -709,6 +714,21 @@ export default class SchemaController {
     });
   }
 
+  async reloadDataIfNeeded() {
+    if (this._dbAdapter.enableSchemaHooks) {
+      return;
+    }
+    const { date, duration } = ttl || {};
+    if (!duration) {
+      return;
+    }
+    const now = Date.now();
+    if (now - date > duration) {
+      ttl.date = now;
+      await this.reloadData({ clearCache: true });
+    }
+  }
+
   reloadData(options: LoadSchemaOptions = { clearCache: false }): Promise<any> {
     if (this.reloadDataPromise && !options.clearCache) {
       return this.reloadDataPromise;
@@ -729,10 +749,11 @@ export default class SchemaController {
     return this.reloadDataPromise;
   }
 
-  getAllClasses(options: LoadSchemaOptions = { clearCache: false }): Promise<Array<Schema>> {
+  async getAllClasses(options: LoadSchemaOptions = { clearCache: false }): Promise<Array<Schema>> {
     if (options.clearCache) {
       return this.setAllClasses();
     }
+    await this.reloadDataIfNeeded();
     const cached = SchemaCache.all();
     if (cached && cached.length) {
       return Promise.resolve(cached);
@@ -1440,6 +1461,7 @@ export default class SchemaController {
 // Returns a promise for a new Schema.
 const load = (dbAdapter: StorageAdapter, options: any): Promise<SchemaController> => {
   const schema = new SchemaController(dbAdapter);
+  ttl.duration = dbAdapter.schemaCacheTtl;
   return schema.reloadData(options).then(() => schema);
 };
 

src/Options/Definitions.js

@@ -971,6 +971,12 @@ module.exports.DatabaseOptions = {
     action: parsers.booleanParser,
     default: false,
   },
+  schemaCacheTtl: {
+    env: 'PARSE_SERVER_DATABASE_SCHEMA_CACHE_TTL',
+    help:
+      'The duration in seconds after which the schema cache expires and will be refetched from the database. Use this option if using multiple Parse Servers instances connected to the same database. A low duration will cause the schema cache to be updated too often, causing unnecessary database reads. A high duration will cause the schema to be updated too rarely, increasing the time required until schema changes propagate to all server instances. This feature can be used as an alternative or in conjunction with the option `enableSchemaHooks`. Default is infinite which means the schema cache never expires.',
+    action: parsers.numberParser('schemaCacheTtl'),
+  },
 };
 module.exports.AuthAdapter = {
   enabled: {

src/Options/docs.js

@@ -548,6 +548,8 @@ export interface DatabaseOptions {
   /* Enables database real-time hooks to update single schema cache. Set to `true` if using multiple Parse Servers instances connected to the same database. Failing to do so will cause a schema change to not propagate to all instances and re-syncing will only happen when the instances restart. To use this feature with MongoDB, a replica set cluster with [change stream](https://docs.mongodb.com/manual/changeStreams/#availability) support is required.
   :DEFAULT: false */
   enableSchemaHooks: ?boolean;
+  /* The duration in seconds after which the schema cache expires and will be refetched from the database. Use this option if using multiple Parse Servers instances connected to the same database. A low duration will cause the schema cache to be updated too often, causing unnecessary database reads. A high duration will cause the schema to be updated too rarely, increasing the time required until schema changes propagate to all server instances. This feature can be used as an alternative or in conjunction with the option `enableSchemaHooks`. Default is infinite which means the schema cache never expires. */
+  schemaCacheTtl: ?number;
 }
 
 export interface AuthAdapter {

src/Options/index.js

@@ -71,6 +71,7 @@ class ParseServer {
     Parse.initialize(appId, javascriptKey || 'unused', masterKey);
     Parse.serverURL = serverURL;
 
+    Config.validateOptions(options);
     const allControllers = controllers.getControllers(options);
     options.state = 'initialized';
     this.config = Config.put(Object.assign({}, options, allControllers));