Adds schema caching capabilities (5s by default) (#2286)

* Adds schema caching capabilities (off by default)

* Use InMemoryCacheAdapter

* Uses proper adapter to generate a cache

* Fix bugs when running disabled cache

* nits

* nits

* Use options object instead of boolean

* Imrpove concurrency of loadSchema

* Adds testing with SCHEMA_CACHE_ON

* Use CacheController instead of generator

- Makes caching SchemaCache use a generated prefix
- Makes clearing the SchemaCache clear only the cached schema keys
- Enable cache by default (ttl 5s)

Author: flovilmart

spec/ParseInstallation.spec.js

@@ -9,13 +9,19 @@ let Parse = require('parse/node').Parse;
 let rest = require('../src/rest');
 let request = require("request");
 
-let config = new Config('test');
-let database = config.database;
+let config;
+let database;
 let defaultColumns = require('../src/Controllers/SchemaController').defaultColumns;
 
 const installationSchema = { fields: Object.assign({}, defaultColumns._Default, defaultColumns._Installation) };
 
 describe('Installations', () => {
+
+  beforeEach(() => {
+    config = new Config('test');
+    database = config.database;
+  });
+
   it_exclude_dbs(['postgres'])('creates an android installation with ids', (done) => {
     var installId = '12345678-abcd-abcd-abcd-123456789abc';
     var device = 'android';

spec/PointerPermissions.spec.js

@@ -4,6 +4,11 @@ var Schema = require('../src/Controllers/SchemaController');
 var Config = require('../src/Config');
 
 describe('Pointer Permissions', () => {
+
+  beforeEach(() => {
+    new Config(Parse.applicationId).database.schemaCache.clear();
+  });
+
   it_exclude_dbs(['postgres'])('should work with find', (done) => {
     let config = new Config(Parse.applicationId);
     let user = new Parse.User();

spec/RestQuery.spec.js

@@ -9,11 +9,17 @@ var querystring = require('querystring');
 var request = require('request');
 var rp = require('request-promise');
 
-var config = new Config('test');
-let database = config.database;
+var config;
+let database;
 var nobody = auth.nobody(config);
 
 describe('rest query', () => {
+
+  beforeEach(() => {
+    config = new Config('test');
+    database = config.database;
+  });
+
   it('basic query', (done) => {
     rest.create(config, nobody, 'TestObject', {}).then(() => {
       return rest.find(config, nobody, 'TestObject', {});

spec/Schema.spec.js

@@ -4,7 +4,7 @@ var Config = require('../src/Config');
 var SchemaController = require('../src/Controllers/SchemaController');
 var dd = require('deep-diff');
 
-var config = new Config('test');
+var config;
 
 var hasAllPODobject = () => {
   var obj = new Parse.Object('HasAllPOD');
@@ -20,6 +20,10 @@ var hasAllPODobject = () => {
 };
 
 describe('SchemaController', () => {
+  beforeEach(() => {
+    config = new Config('test');
+  });
+
   it('can validate one object', (done) => {
     config.database.loadSchema().then((schema) => {
       return schema.validateObject('TestObject', {a: 1, b: 'yo', c: false});

spec/helper.js

@@ -59,7 +59,7 @@ var defaultConfiguration = {
     myoauth: {
       module: path.resolve(__dirname, "myoauth") // relative path as it's run from src
     }
-  },
+  }
 };
 
 let openConnections = {};

spec/schemas.spec.js

@@ -116,6 +116,11 @@ var masterKeyHeaders = {
 };
 
 describe('schemas', () => {
+
+  beforeEach(() => {
+     config.database.schemaCache.clear();
+  });
+
   it('requires the master key to get all schemas', (done) => {
     request.get({
       url: 'http://localhost:8378/1/schemas',

src/Config.js

@@ -3,6 +3,8 @@
 // mount is the URL for the root of the API; includes http, domain, etc.
 
 import AppCache from './cache';
+import SchemaCache from './Controllers/SchemaCache';
+import DatabaseController from './Controllers/DatabaseController';
 
 function removeTrailingSlash(str) {
   if (!str) {
@@ -32,7 +34,14 @@ export class Config {
     this.fileKey = cacheInfo.fileKey;
     this.facebookAppIds = cacheInfo.facebookAppIds;
     this.allowClientClassCreation = cacheInfo.allowClientClassCreation;
-    this.database = cacheInfo.databaseController;
+
+    // Create a new DatabaseController per request
+    if (cacheInfo.databaseController) {
+      const schemaCache = new SchemaCache(cacheInfo.cacheController, cacheInfo.schemaCacheTTL);
+      this.database = new DatabaseController(cacheInfo.databaseController.adapter, schemaCache);
+    }
+
+    this.schemaCacheTTL = cacheInfo.schemaCacheTTL;
 
     this.serverURL = cacheInfo.serverURL;
     this.publicServerURL = removeTrailingSlash(cacheInfo.publicServerURL);

src/Controllers/CacheController.js

@@ -13,9 +13,10 @@ function joinKeys(...keys) {
  * eg "Role" or "Session"
  */
 export class SubCache {
-  constructor(prefix, cacheController) {
+  constructor(prefix, cacheController, ttl) {
     this.prefix = prefix;
     this.cache = cacheController;
+    this.ttl = ttl;
   }
 
   get(key) {

src/Controllers/DatabaseController.js

@@ -7,7 +7,8 @@ import _         from 'lodash';
 var mongodb = require('mongodb');
 var Parse = require('parse/node').Parse;
 
-var SchemaController = require('../Controllers/SchemaController');
+var SchemaController = require('./SchemaController');
+
 const deepcopy = require('deepcopy');
 
 function addWriteACL(query, acl) {
@@ -80,9 +81,9 @@ const validateQuery = query => {
   });
 }
 
-function DatabaseController(adapter) {
+function DatabaseController(adapter, schemaCache) {
   this.adapter = adapter;
-
+  this.schemaCache = schemaCache;
   // We don't want a mutable this.schema, because then you could have
   // one request that uses different schemas for different parts of
   // it. Instead, use loadSchema to get a schema.
@@ -107,9 +108,9 @@ DatabaseController.prototype.validateClassName = function(className) {
 };
 
 // Returns a promise for a schemaController.
-DatabaseController.prototype.loadSchema = function() {
+DatabaseController.prototype.loadSchema = function(options = {clearCache: false}) {
   if (!this.schemaPromise) {
-    this.schemaPromise = SchemaController.load(this.adapter);
+    this.schemaPromise = SchemaController.load(this.adapter, this.schemaCache, options);
     this.schemaPromise.then(() => delete this.schemaPromise,
                              () => delete this.schemaPromise);
   }
@@ -805,8 +806,8 @@ const untransformObjectACL = ({_rperm, _wperm, ...output}) => {
 }
 
 DatabaseController.prototype.deleteSchema = function(className) {
-  return this.loadSchema()
-  .then(schemaController => schemaController.getOneSchema(className))
+  return this.loadSchema(true)
+  .then(schemaController => schemaController.getOneSchema(className, true))
   .catch(error => {
     if (error === undefined) {
       return { fields: {} };

src/Controllers/SchemaCache.js

@@ -0,0 +1,68 @@
+const MAIN_SCHEMA = "__MAIN_SCHEMA";
+const SCHEMA_CACHE_PREFIX = "__SCHEMA";
+const ALL_KEYS = "__ALL_KEYS";
+
+import { randomString } from '../cryptoUtils';
+
+export default class SchemaCache {
+  cache: Object;
+
+  constructor(cacheController, ttl = 30) {
+    this.ttl = ttl;
+    if (typeof ttl == 'string') {
+      this.ttl = parseInt(ttl);
+    }
+    this.cache = cacheController;
+    this.prefix = SCHEMA_CACHE_PREFIX+randomString(20);
+  }
+
+  put(key, value) {
+    return this.cache.get(this.prefix+ALL_KEYS).then((allKeys) => {
+      allKeys = allKeys || {};
+      allKeys[key] = true;
+      return Promise.all([this.cache.put(this.prefix+ALL_KEYS, allKeys, this.ttl), this.cache.put(key, value, this.ttl)]);
+    });
+  }
+
+  getAllClasses() {
+    if (!this.ttl) {
+      return Promise.resolve(null);
+    }
+    return this.cache.get(this.prefix+MAIN_SCHEMA);
+  }
+
+  setAllClasses(schema) {
+    if (!this.ttl) {
+      return Promise.resolve(null);
+    }
+    return this.put(this.prefix+MAIN_SCHEMA, schema);
+  }
+
+  setOneSchema(className, schema) {
+    if (!this.ttl) {
+      return Promise.resolve(null);
+    }
+    return this.put(this.prefix+className, schema);
+  }
+
+  getOneSchema(className) {
+    if (!this.ttl) {
+      return Promise.resolve(null);
+    }
+    return this.cache.get(this.prefix+className);
+  }
+
+  clear() {
+    // That clears all caches...
+    let promise = Promise.resolve();
+    return this.cache.get(this.prefix+ALL_KEYS).then((allKeys) => {
+      if (!allKeys) {
+        return;
+      }
+      let promises = Object.keys(allKeys).map((key) => {
+        return this.cache.del(key);
+      });
+      return Promise.all(promises);
+    });
+  }
+}