From 3ea3ca35973b7e648d9e23298fe1bc4c468ac5c3 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Thu, 14 Jul 2016 15:23:51 -0400 Subject: [PATCH 01/10] Adds schema caching capabilities (off by default) --- spec/ParseInstallation.spec.js | 5 ++ spec/PointerPermissions.spec.js | 5 ++ spec/Schema.spec.js | 4 ++ spec/schemas.spec.js | 5 ++ src/Config.js | 8 +++- src/Controllers/DatabaseController.js | 12 ++--- src/Controllers/SchemaCache.js | 50 +++++++++++++++++++ src/Controllers/SchemaController.js | 69 ++++++++++++++++++++------- src/ParseServer.js | 7 ++- src/Routers/SchemasRouter.js | 12 ++--- src/cli/cli-definitions.js | 5 ++ 11 files changed, 150 insertions(+), 32 deletions(-) create mode 100644 src/Controllers/SchemaCache.js diff --git a/spec/ParseInstallation.spec.js b/spec/ParseInstallation.spec.js index 8681179adb..ae3b531ab0 100644 --- a/spec/ParseInstallation.spec.js +++ b/spec/ParseInstallation.spec.js @@ -16,6 +16,11 @@ let defaultColumns = require('../src/Controllers/SchemaController').defaultColum const installationSchema = { fields: Object.assign({}, defaultColumns._Default, defaultColumns._Installation) }; describe('Installations', () => { + + beforeEach(() => { + config.database.schemaCache.reset(); + }); + it_exclude_dbs(['postgres'])('creates an android installation with ids', (done) => { var installId = '12345678-abcd-abcd-abcd-123456789abc'; var device = 'android'; diff --git a/spec/PointerPermissions.spec.js b/spec/PointerPermissions.spec.js index ece6d3a17b..b60ba576f3 100644 --- a/spec/PointerPermissions.spec.js +++ b/spec/PointerPermissions.spec.js @@ -4,6 +4,11 @@ var Schema = require('../src/Controllers/SchemaController'); var Config = require('../src/Config'); describe('Pointer Permissions', () => { + + beforeEach(() => { + new Config(Parse.applicationId).database.schemaCache.reset(); + }); + it_exclude_dbs(['postgres'])('should work with find', (done) => { let config = new Config(Parse.applicationId); let user = new Parse.User(); diff --git a/spec/Schema.spec.js b/spec/Schema.spec.js index 0a6b17b3ea..ee2fa25b91 100644 --- a/spec/Schema.spec.js +++ b/spec/Schema.spec.js @@ -20,6 +20,10 @@ var hasAllPODobject = () => { }; describe('SchemaController', () => { + beforeEach(() => { + config.database.schemaCache.reset(); + }); + it('can validate one object', (done) => { config.database.loadSchema().then((schema) => { return schema.validateObject('TestObject', {a: 1, b: 'yo', c: false}); diff --git a/spec/schemas.spec.js b/spec/schemas.spec.js index 37276b400a..8f1e67a2b9 100644 --- a/spec/schemas.spec.js +++ b/spec/schemas.spec.js @@ -116,6 +116,11 @@ var masterKeyHeaders = { }; describe('schemas', () => { + + beforeEach(() => { + config.database.schemaCache.reset(); + }) + it('requires the master key to get all schemas', (done) => { request.get({ url: 'http://localhost:8378/1/schemas', diff --git a/src/Config.js b/src/Config.js index 963022456c..0c91ed20bb 100644 --- a/src/Config.js +++ b/src/Config.js @@ -3,6 +3,8 @@ // mount is the URL for the root of the API; includes http, domain, etc. import AppCache from './cache'; +import SchemaCache from './Controllers/SchemaCache'; +import DatabaseController from './Controllers/DatabaseController'; function removeTrailingSlash(str) { if (!str) { @@ -32,7 +34,11 @@ export class Config { this.fileKey = cacheInfo.fileKey; this.facebookAppIds = cacheInfo.facebookAppIds; this.allowClientClassCreation = cacheInfo.allowClientClassCreation; - this.database = cacheInfo.databaseController; + + // Create a new DatabaseController per request + if (cacheInfo.databaseController) { + this.database = new DatabaseController(cacheInfo.databaseController.adapter, new SchemaCache(applicationId, cacheInfo.schemaCacheTTL)); + } this.serverURL = cacheInfo.serverURL; this.publicServerURL = removeTrailingSlash(cacheInfo.publicServerURL); diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index cf64caa0bf..4c8cfde22d 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -80,9 +80,9 @@ const validateQuery = query => { }); } -function DatabaseController(adapter) { +function DatabaseController(adapter, schemaCache) { this.adapter = adapter; - + this.schemaCache = schemaCache; // We don't want a mutable this.schema, because then you could have // one request that uses different schemas for different parts of // it. Instead, use loadSchema to get a schema. @@ -107,9 +107,9 @@ DatabaseController.prototype.validateClassName = function(className) { }; // Returns a promise for a schemaController. -DatabaseController.prototype.loadSchema = function() { +DatabaseController.prototype.loadSchema = function(force = false) { if (!this.schemaPromise) { - this.schemaPromise = SchemaController.load(this.adapter); + this.schemaPromise = SchemaController.load(this.adapter, this.schemaCache, force); this.schemaPromise.then(() => delete this.schemaPromise, () => delete this.schemaPromise); } @@ -805,8 +805,8 @@ const untransformObjectACL = ({_rperm, _wperm, ...output}) => { } DatabaseController.prototype.deleteSchema = function(className) { - return this.loadSchema() - .then(schemaController => schemaController.getOneSchema(className)) + return this.loadSchema(true) + .then(schemaController => schemaController.getOneSchema(className, true)) .catch(error => { if (error === undefined) { return { fields: {} }; diff --git a/src/Controllers/SchemaCache.js b/src/Controllers/SchemaCache.js new file mode 100644 index 0000000000..294d442026 --- /dev/null +++ b/src/Controllers/SchemaCache.js @@ -0,0 +1,50 @@ +import Parse from 'parse/node'; +import LRU from 'lru-cache'; + +let MAIN_SCHEMA = "__MAIN_SCHEMA"; + +export default class SchemaCache { + cache: Object; + + constructor(appId: string, timeout: number = 0, maxSize: number = 10000) { + this.appId = appId; + this.timeout = timeout; + this.maxSize = maxSize; + this.cache = new LRU({ + max: maxSize, + maxAge: timeout + }); + } + + get() { + if (this.timeout <= 0) { + return; + } + return this.cache.get(this.appId+MAIN_SCHEMA); + } + + set(schema) { + if (this.timeout <= 0) { + return; + } + this.cache.set(this.appId+MAIN_SCHEMA, schema); + } + + setOneSchema(className, schema) { + if (this.timeout <= 0) { + return; + } + this.cache.set(this.appId+className, schema); + } + + getOneSchema(className) { + if (this.timeout <= 0) { + return; + } + return this.cache.get(this.appId+className); + } + + reset() { + this.cache.reset(); + } +} diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 8481f5344d..7409f2d29d 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -273,19 +273,22 @@ class SchemaController { data; perms; - constructor(databaseAdapter) { + constructor(databaseAdapter, schemaCache) { this._dbAdapter = databaseAdapter; - + this._cache = schemaCache; // this.data[className][fieldName] tells you the type of that field, in mongo format this.data = {}; // this.perms[className][operation] tells you the acl-style permissions this.perms = {}; } - reloadData() { + reloadData(clearCache = false) { this.data = {}; this.perms = {}; - return this.getAllClasses() + if (clearCache) { + this._cache.reset(); + } + return this.getAllClasses(clearCache) .then(allSchemas => { allSchemas.forEach(schema => { this.data[schema.className] = injectDefaultSchema(schema).fields; @@ -303,17 +306,39 @@ class SchemaController { }); } - getAllClasses() { + getAllClasses(clearCache = false) { + if (clearCache) { + this._cache.reset(); + } + let allClasses = this._cache.get(); + if (allClasses && allClasses.length && !clearCache) { + return Promise.resolve(allClasses); + } return this._dbAdapter.getAllClasses() - .then(allSchemas => allSchemas.map(injectDefaultSchema)); + .then(allSchemas => allSchemas.map(injectDefaultSchema)) + .then(allSchemas => { + this._cache.set(allSchemas); + return allSchemas; + }) } - getOneSchema(className, allowVolatileClasses = false) { + getOneSchema(className, allowVolatileClasses = false, clearCache) { + if (clearCache) { + this._cache.reset(); + } + let cached = this._cache.getOneSchema(className); + if (cached && !clearCache) { + return Promise.resolve(cached); + } if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) { return Promise.resolve(this.data[className]); } return this._dbAdapter.getClass(className) .then(injectDefaultSchema) + .then((result) => { + this._cache.setOneSchema(className, result); + return result; + }) } // Create a new class that includes the three default fields. @@ -331,6 +356,10 @@ class SchemaController { return this._dbAdapter.createClass(className, convertSchemaToAdapterSchema({ fields, classLevelPermissions, className })) .then(convertAdapterSchemaToParseSchema) + .then((res) => { + this._cache.reset(); + return res; + }) .catch(error => { if (error && error.code === Parse.Error.DUPLICATE_VALUE) { throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} already exists.`); @@ -376,7 +405,7 @@ class SchemaController { }); return Promise.all(deletePromises) // Delete Everything - .then(() => this.reloadData()) // Reload our Schema, so we have all the new values + .then(() => this.reloadData(true)) // Reload our Schema, so we have all the new values .then(() => { let promises = insertedFields.map(fieldName => { const type = submittedFields[fieldName]; @@ -410,13 +439,13 @@ class SchemaController { // We don't have this class. Update the schema return this.addClassIfNotExists(className) // The schema update succeeded. Reload the schema - .then(() => this.reloadData()) + .then(() => this.reloadData(true)) .catch(error => { // The schema update failed. This can be okay - it might // have failed because there's a race condition and a different // client is making the exact same schema update that we want. // So just reload the schema. - return this.reloadData(); + return this.reloadData(true); }) .then(() => { // Ensure that the schema now validates @@ -486,7 +515,7 @@ class SchemaController { } validateCLP(perms, newSchema); return this._dbAdapter.setClassLevelPermissions(className, perms) - .then(() => this.reloadData()); + .then(() => this.reloadData(true)); } // Returns a promise that resolves successfully to the new schema @@ -521,23 +550,26 @@ class SchemaController { `schema mismatch for ${className}.${fieldName}; expected ${expectedType.type || expectedType} but got ${type.type}` ); } + return this; } return this._dbAdapter.addFieldIfNotExists(className, fieldName, type).then(() => { // The update succeeded. Reload the schema - return this.reloadData(); + return this.reloadData(true); }, error => { //TODO: introspect the error and only reload if the error is one for which is makes sense to reload // The update failed. This can be okay - it might have been a race // condition where another client updated the schema in the same // way that we wanted to. So, just reload the schema - return this.reloadData(); + return this.reloadData(true); }).then(error => { // Ensure that the schema now validates if (!dbTypeMatchesObjectType(this.getExpectedType(className, fieldName), type)) { throw new Parse.Error(Parse.Error.INVALID_JSON, `Could not add field ${fieldName}`); } + // Remove the cached schema + this._cache.reset(); return this; }); }); @@ -562,7 +594,7 @@ class SchemaController { throw new Parse.Error(136, `field ${fieldName} cannot be changed`); } - return this.getOneSchema(className) + return this.getOneSchema(className, false, true) .catch(error => { if (error === undefined) { throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} does not exist.`); @@ -579,8 +611,9 @@ class SchemaController { return database.adapter.deleteFields(className, schema, [fieldName]) .then(() => database.adapter.deleteClass(`_Join:${fieldName}:${className}`)); } - return database.adapter.deleteFields(className, schema, [fieldName]); + }).then(() => { + this._cache.reset(); }); } @@ -711,9 +744,9 @@ class SchemaController { } // Returns a promise for a new Schema. -const load = dbAdapter => { - let schema = new SchemaController(dbAdapter); - return schema.reloadData().then(() => schema); +const load = (dbAdapter, schemaCache, clearCache) => { + let schema = new SchemaController(dbAdapter, schemaCache); + return schema.reloadData(clearCache).then(() => schema); } // Builds a new schema (in schema API response format) out of an diff --git a/src/ParseServer.js b/src/ParseServer.js index 5eb3f1836a..4766b043b9 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -55,6 +55,7 @@ import { UsersRouter } from './Routers/UsersRouter'; import { PurgeRouter } from './Routers/PurgeRouter'; import DatabaseController from './Controllers/DatabaseController'; +import SchemaCache from './Controllers/SchemaCache'; const SchemaController = require('./Controllers/SchemaController'); import ParsePushAdapter from 'parse-server-push-adapter'; import MongoStorageAdapter from './Adapters/Storage/Mongo/MongoStorageAdapter'; @@ -139,6 +140,7 @@ class ParseServer { expireInactiveSessions = true, verbose = false, revokeSessionOnPasswordReset = true, + schemaCacheTTL = -1, // -1 = no cache __indexBuildCompletionCallbackForTests = () => {}, }) { // Initialize the node client SDK automatically @@ -197,7 +199,8 @@ class ParseServer { const userController = new UserController(emailControllerAdapter, appId, { verifyUserEmails }); const liveQueryController = new LiveQueryController(liveQuery); const cacheController = new CacheController(cacheControllerAdapter, appId); - const databaseController = new DatabaseController(databaseAdapter); + const schemaCache = new SchemaCache(appId, schemaCacheTTL); + const databaseController = new DatabaseController(databaseAdapter, schemaCache); const hooksController = new HooksController(appId, databaseController, webhookKey); const analyticsController = new AnalyticsController(analyticsControllerAdapter); @@ -254,6 +257,8 @@ class ParseServer { jsonLogs, revokeSessionOnPasswordReset, databaseController, + schemaCache, + schemaCacheTTL }); // To maintain compatibility. TODO: Remove in some version that breaks backwards compatability diff --git a/src/Routers/SchemasRouter.js b/src/Routers/SchemasRouter.js index 4024d0a2db..8a67a5af0c 100644 --- a/src/Routers/SchemasRouter.js +++ b/src/Routers/SchemasRouter.js @@ -15,15 +15,15 @@ function classNameMismatchResponse(bodyClass, pathClass) { } function getAllSchemas(req) { - return req.config.database.loadSchema() - .then(schemaController => schemaController.getAllClasses()) + return req.config.database.loadSchema(true) + .then(schemaController => schemaController.getAllClasses(true)) .then(schemas => ({ response: { results: schemas } })); } function getOneSchema(req) { const className = req.params.className; - return req.config.database.loadSchema() - .then(schemaController => schemaController.getOneSchema(className)) + return req.config.database.loadSchema(true) + .then(schemaController => schemaController.getOneSchema(className, true)) .then(schema => ({ response: schema })) .catch(error => { if (error === undefined) { @@ -46,7 +46,7 @@ function createSchema(req) { throw new Parse.Error(135, `POST ${req.path} needs a class name.`); } - return req.config.database.loadSchema() + return req.config.database.loadSchema(true) .then(schema => schema.addClassIfNotExists(className, req.body.fields, req.body.classLevelPermissions)) .then(schema => ({ response: schema })); } @@ -59,7 +59,7 @@ function modifySchema(req) { let submittedFields = req.body.fields || {}; let className = req.params.className; - return req.config.database.loadSchema() + return req.config.database.loadSchema(true) .then(schema => schema.updateClass(className, submittedFields, req.body.classLevelPermissions, req.config.database)) .then(result => ({response: result})); } diff --git a/src/cli/cli-definitions.js b/src/cli/cli-definitions.js index 3fc1e5dccd..db34ee49e6 100644 --- a/src/cli/cli-definitions.js +++ b/src/cli/cli-definitions.js @@ -197,5 +197,10 @@ export default { env: "PARSE_SERVER_REVOKE_SESSION_ON_PASSWORD_RESET", help: "When a user changes their password, either through the reset password email or while logged in, all sessions are revoked if this is true. Set to false if you don't want to revoke sessions.", action: booleanParser + }, + "schemaCacheTTL": { + env: "PARSE_SERVER_SCHEMA_CACHE_TTL", + help: "The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to -1; disabled.", + action: numberParser("schemaCacheTTL"), } }; From faaaff37d1d8658a82c6671aa951f5853239bc6e Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Fri, 15 Jul 2016 11:16:34 -0400 Subject: [PATCH 02/10] Use InMemoryCacheAdapter --- spec/ParseInstallation.spec.js | 2 +- spec/PointerPermissions.spec.js | 2 +- spec/Schema.spec.js | 2 +- spec/schemas.spec.js | 4 +- src/Config.js | 2 +- src/Controllers/CacheController.js | 3 +- src/Controllers/DatabaseController.js | 4 +- src/Controllers/SchemaCache.js | 46 ++++++++++---------- src/Controllers/SchemaController.js | 60 ++++++++++++++------------- src/ParseServer.js | 4 +- 10 files changed, 64 insertions(+), 65 deletions(-) diff --git a/spec/ParseInstallation.spec.js b/spec/ParseInstallation.spec.js index ae3b531ab0..c1730985c0 100644 --- a/spec/ParseInstallation.spec.js +++ b/spec/ParseInstallation.spec.js @@ -18,7 +18,7 @@ const installationSchema = { fields: Object.assign({}, defaultColumns._Default, describe('Installations', () => { beforeEach(() => { - config.database.schemaCache.reset(); + config.database.schemaCache.clear(); }); it_exclude_dbs(['postgres'])('creates an android installation with ids', (done) => { diff --git a/spec/PointerPermissions.spec.js b/spec/PointerPermissions.spec.js index b60ba576f3..4a1b048f64 100644 --- a/spec/PointerPermissions.spec.js +++ b/spec/PointerPermissions.spec.js @@ -6,7 +6,7 @@ var Config = require('../src/Config'); describe('Pointer Permissions', () => { beforeEach(() => { - new Config(Parse.applicationId).database.schemaCache.reset(); + new Config(Parse.applicationId).database.schemaCache.clear(); }); it_exclude_dbs(['postgres'])('should work with find', (done) => { diff --git a/spec/Schema.spec.js b/spec/Schema.spec.js index ee2fa25b91..1752cca671 100644 --- a/spec/Schema.spec.js +++ b/spec/Schema.spec.js @@ -21,7 +21,7 @@ var hasAllPODobject = () => { describe('SchemaController', () => { beforeEach(() => { - config.database.schemaCache.reset(); + config.database.schemaCache.clear(); }); it('can validate one object', (done) => { diff --git a/spec/schemas.spec.js b/spec/schemas.spec.js index 8f1e67a2b9..3ad733a227 100644 --- a/spec/schemas.spec.js +++ b/spec/schemas.spec.js @@ -118,8 +118,8 @@ var masterKeyHeaders = { describe('schemas', () => { beforeEach(() => { - config.database.schemaCache.reset(); - }) + config.database.schemaCache.clear(); + }); it('requires the master key to get all schemas', (done) => { request.get({ diff --git a/src/Config.js b/src/Config.js index 0c91ed20bb..a484648d87 100644 --- a/src/Config.js +++ b/src/Config.js @@ -37,7 +37,7 @@ export class Config { // Create a new DatabaseController per request if (cacheInfo.databaseController) { - this.database = new DatabaseController(cacheInfo.databaseController.adapter, new SchemaCache(applicationId, cacheInfo.schemaCacheTTL)); + this.database = new DatabaseController(cacheInfo.databaseController.adapter, new SchemaCache(cacheInfo.schemaCacheTTL)); } this.serverURL = cacheInfo.serverURL; diff --git a/src/Controllers/CacheController.js b/src/Controllers/CacheController.js index 27dc4936f7..a6c88e9362 100644 --- a/src/Controllers/CacheController.js +++ b/src/Controllers/CacheController.js @@ -13,9 +13,10 @@ function joinKeys(...keys) { * eg "Role" or "Session" */ export class SubCache { - constructor(prefix, cacheController) { + constructor(prefix, cacheController, ttl) { this.prefix = prefix; this.cache = cacheController; + this.ttl = ttl; } get(key) { diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 4c8cfde22d..1adefbe835 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -7,7 +7,9 @@ import _ from 'lodash'; var mongodb = require('mongodb'); var Parse = require('parse/node').Parse; -var SchemaController = require('../Controllers/SchemaController'); +var SchemaController = require('./SchemaController'); +import SchemaCache from './SchemaCache'; + const deepcopy = require('deepcopy'); function addWriteACL(query, acl) { diff --git a/src/Controllers/SchemaCache.js b/src/Controllers/SchemaCache.js index 294d442026..f97f35e4d2 100644 --- a/src/Controllers/SchemaCache.js +++ b/src/Controllers/SchemaCache.js @@ -1,50 +1,46 @@ -import Parse from 'parse/node'; -import LRU from 'lru-cache'; - -let MAIN_SCHEMA = "__MAIN_SCHEMA"; +import { InMemoryCacheAdapter } from '../Adapters/Cache/InMemoryCacheAdapter'; +const CACHED_KEYS = "__CACHED_KEYS"; +const MAIN_SCHEMA = "__MAIN_SCHEMA"; +const SCHEMA_CACHE_PREFIX = "__SCHEMA"; export default class SchemaCache { cache: Object; - constructor(appId: string, timeout: number = 0, maxSize: number = 10000) { - this.appId = appId; - this.timeout = timeout; - this.maxSize = maxSize; - this.cache = new LRU({ - max: maxSize, - maxAge: timeout - }); - } + constructor(ttl) { + this.ttl = ttl; + this.cache = new InMemoryCacheAdapter({ ttl }); +} - get() { - if (this.timeout <= 0) { + getAllClasses() { + if (this.ttl <= 0) { return; } - return this.cache.get(this.appId+MAIN_SCHEMA); + return this.cache.get(SCHEMA_CACHE_PREFIX+MAIN_SCHEMA); } - set(schema) { - if (this.timeout <= 0) { + setAllClasses(schema) { + if (this.ttl <= 0) { return; } - this.cache.set(this.appId+MAIN_SCHEMA, schema); + this.cache.put(SCHEMA_CACHE_PREFIX+MAIN_SCHEMA, schema, this.ttl); } setOneSchema(className, schema) { - if (this.timeout <= 0) { + if (this.ttl <= 0) { return; } - this.cache.set(this.appId+className, schema); + this.cache.put(SCHEMA_CACHE_PREFIX+className, schema, this.ttl); } getOneSchema(className) { - if (this.timeout <= 0) { + if (this.ttl <= 0) { return; } - return this.cache.get(this.appId+className); + return this.cache.get(SCHEMA_CACHE_PREFIX+className); } - reset() { - this.cache.reset(); + clear() { + // That clears all caches... + this.cache.clear(); } } diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 7409f2d29d..e68a199932 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -286,7 +286,7 @@ class SchemaController { this.data = {}; this.perms = {}; if (clearCache) { - this._cache.reset(); + this._cache.clear(); } return this.getAllClasses(clearCache) .then(allSchemas => { @@ -308,37 +308,39 @@ class SchemaController { getAllClasses(clearCache = false) { if (clearCache) { - this._cache.reset(); + this._cache.clear(); } - let allClasses = this._cache.get(); - if (allClasses && allClasses.length && !clearCache) { - return Promise.resolve(allClasses); - } - return this._dbAdapter.getAllClasses() - .then(allSchemas => allSchemas.map(injectDefaultSchema)) - .then(allSchemas => { - this._cache.set(allSchemas); - return allSchemas; - }) + return this._cache.getAllClasses().then((allClasses) => { + if (allClasses && allClasses.length && !clearCache) { + return Promise.resolve(allClasses); + } + return this._dbAdapter.getAllClasses() + .then(allSchemas => allSchemas.map(injectDefaultSchema)) + .then(allSchemas => { + this._cache.setAllClasses(allSchemas); + return allSchemas; + }) + }); } getOneSchema(className, allowVolatileClasses = false, clearCache) { if (clearCache) { - this._cache.reset(); - } - let cached = this._cache.getOneSchema(className); - if (cached && !clearCache) { - return Promise.resolve(cached); + this._cache.clear(); } - if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) { - return Promise.resolve(this.data[className]); - } - return this._dbAdapter.getClass(className) - .then(injectDefaultSchema) - .then((result) => { - this._cache.setOneSchema(className, result); - return result; - }) + return this._cache.getOneSchema(className).then((cached) => { + if (cached && !clearCache) { + return Promise.resolve(cached); + } + if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) { + return Promise.resolve(this.data[className]); + } + return this._dbAdapter.getClass(className) + .then(injectDefaultSchema) + .then((result) => { + this._cache.setOneSchema(className, result); + return result; + }); + }); } // Create a new class that includes the three default fields. @@ -357,7 +359,7 @@ class SchemaController { return this._dbAdapter.createClass(className, convertSchemaToAdapterSchema({ fields, classLevelPermissions, className })) .then(convertAdapterSchemaToParseSchema) .then((res) => { - this._cache.reset(); + this._cache.clear(); return res; }) .catch(error => { @@ -569,7 +571,7 @@ class SchemaController { throw new Parse.Error(Parse.Error.INVALID_JSON, `Could not add field ${fieldName}`); } // Remove the cached schema - this._cache.reset(); + this._cache.clear(); return this; }); }); @@ -613,7 +615,7 @@ class SchemaController { } return database.adapter.deleteFields(className, schema, [fieldName]); }).then(() => { - this._cache.reset(); + this._cache.clear(); }); } diff --git a/src/ParseServer.js b/src/ParseServer.js index 4766b043b9..59382ae4e1 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -199,8 +199,7 @@ class ParseServer { const userController = new UserController(emailControllerAdapter, appId, { verifyUserEmails }); const liveQueryController = new LiveQueryController(liveQuery); const cacheController = new CacheController(cacheControllerAdapter, appId); - const schemaCache = new SchemaCache(appId, schemaCacheTTL); - const databaseController = new DatabaseController(databaseAdapter, schemaCache); + const databaseController = new DatabaseController(databaseAdapter, new SchemaCache(schemaCacheTTL)); const hooksController = new HooksController(appId, databaseController, webhookKey); const analyticsController = new AnalyticsController(analyticsControllerAdapter); @@ -257,7 +256,6 @@ class ParseServer { jsonLogs, revokeSessionOnPasswordReset, databaseController, - schemaCache, schemaCacheTTL }); From cbf3f4475385491865663c4991a9367ede1cc52b Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Fri, 15 Jul 2016 11:28:57 -0400 Subject: [PATCH 03/10] Uses proper adapter to generate a cache --- src/Config.js | 2 +- src/Controllers/SchemaCache.js | 6 ++---- src/ParseServer.js | 9 +++++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Config.js b/src/Config.js index a484648d87..89ea4cad82 100644 --- a/src/Config.js +++ b/src/Config.js @@ -37,7 +37,7 @@ export class Config { // Create a new DatabaseController per request if (cacheInfo.databaseController) { - this.database = new DatabaseController(cacheInfo.databaseController.adapter, new SchemaCache(cacheInfo.schemaCacheTTL)); + this.database = new DatabaseController(cacheInfo.databaseController.adapter, cacheInfo.createSchemaCache()); } this.serverURL = cacheInfo.serverURL; diff --git a/src/Controllers/SchemaCache.js b/src/Controllers/SchemaCache.js index f97f35e4d2..b3f6db2bc9 100644 --- a/src/Controllers/SchemaCache.js +++ b/src/Controllers/SchemaCache.js @@ -1,14 +1,12 @@ -import { InMemoryCacheAdapter } from '../Adapters/Cache/InMemoryCacheAdapter'; - const CACHED_KEYS = "__CACHED_KEYS"; const MAIN_SCHEMA = "__MAIN_SCHEMA"; const SCHEMA_CACHE_PREFIX = "__SCHEMA"; export default class SchemaCache { cache: Object; - constructor(ttl) { + constructor(adapter, ttl) { this.ttl = ttl; - this.cache = new InMemoryCacheAdapter({ ttl }); + this.cache = adapter; } getAllClasses() { diff --git a/src/ParseServer.js b/src/ParseServer.js index 59382ae4e1..de7cc75227 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -191,6 +191,10 @@ class ParseServer { const cacheControllerAdapter = loadAdapter(cacheAdapter, InMemoryCacheAdapter, {appId: appId}); const analyticsControllerAdapter = loadAdapter(analyticsAdapter, AnalyticsAdapter); + const createSchemaCache = function() { + let adapter = loadAdapter(cacheAdapter, InMemoryCacheAdapter, {appId: appId, ttl: schemaCacheTTL}); + return new SchemaCache(adapter, schemaCacheTTL); + } // We pass the options and the base class for the adatper, // Note that passing an instance would work too const filesController = new FilesController(filesControllerAdapter, appId); @@ -199,7 +203,7 @@ class ParseServer { const userController = new UserController(emailControllerAdapter, appId, { verifyUserEmails }); const liveQueryController = new LiveQueryController(liveQuery); const cacheController = new CacheController(cacheControllerAdapter, appId); - const databaseController = new DatabaseController(databaseAdapter, new SchemaCache(schemaCacheTTL)); + const databaseController = new DatabaseController(databaseAdapter, createSchemaCache()); const hooksController = new HooksController(appId, databaseController, webhookKey); const analyticsController = new AnalyticsController(analyticsControllerAdapter); @@ -256,7 +260,8 @@ class ParseServer { jsonLogs, revokeSessionOnPasswordReset, databaseController, - schemaCacheTTL + schemaCacheTTL, + createSchemaCache }); // To maintain compatibility. TODO: Remove in some version that breaks backwards compatability From e835d9f7d1a838f230dce0df0297e8c12da636b3 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Fri, 15 Jul 2016 11:44:35 -0400 Subject: [PATCH 04/10] Fix bugs when running disabled cache --- src/Controllers/SchemaCache.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Controllers/SchemaCache.js b/src/Controllers/SchemaCache.js index b3f6db2bc9..a7341d9fd5 100644 --- a/src/Controllers/SchemaCache.js +++ b/src/Controllers/SchemaCache.js @@ -11,28 +11,28 @@ export default class SchemaCache { getAllClasses() { if (this.ttl <= 0) { - return; + return Promise.resolve(null); } return this.cache.get(SCHEMA_CACHE_PREFIX+MAIN_SCHEMA); } setAllClasses(schema) { if (this.ttl <= 0) { - return; + return Promise.resolve(null); } this.cache.put(SCHEMA_CACHE_PREFIX+MAIN_SCHEMA, schema, this.ttl); } setOneSchema(className, schema) { if (this.ttl <= 0) { - return; + return Promise.resolve(null); } this.cache.put(SCHEMA_CACHE_PREFIX+className, schema, this.ttl); } getOneSchema(className) { if (this.ttl <= 0) { - return; + return Promise.resolve(null); } return this.cache.get(SCHEMA_CACHE_PREFIX+className); } From cfa934731382d339eae2f01805f73c0664982ea1 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Fri, 15 Jul 2016 11:49:46 -0400 Subject: [PATCH 05/10] nits --- src/Controllers/SchemaCache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Controllers/SchemaCache.js b/src/Controllers/SchemaCache.js index a7341d9fd5..0dfd953b2c 100644 --- a/src/Controllers/SchemaCache.js +++ b/src/Controllers/SchemaCache.js @@ -1,6 +1,6 @@ -const CACHED_KEYS = "__CACHED_KEYS"; const MAIN_SCHEMA = "__MAIN_SCHEMA"; const SCHEMA_CACHE_PREFIX = "__SCHEMA"; + export default class SchemaCache { cache: Object; From 190b9833daf2506f653d57cce664cd078ddd18ef Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Fri, 15 Jul 2016 12:04:54 -0400 Subject: [PATCH 06/10] nits --- src/Controllers/SchemaCache.js | 8 ++++---- src/ParseServer.js | 2 +- src/cli/cli-definitions.js | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Controllers/SchemaCache.js b/src/Controllers/SchemaCache.js index 0dfd953b2c..f9dfa0b508 100644 --- a/src/Controllers/SchemaCache.js +++ b/src/Controllers/SchemaCache.js @@ -10,28 +10,28 @@ export default class SchemaCache { } getAllClasses() { - if (this.ttl <= 0) { + if (!this.ttl) { return Promise.resolve(null); } return this.cache.get(SCHEMA_CACHE_PREFIX+MAIN_SCHEMA); } setAllClasses(schema) { - if (this.ttl <= 0) { + if (!this.ttl) { return Promise.resolve(null); } this.cache.put(SCHEMA_CACHE_PREFIX+MAIN_SCHEMA, schema, this.ttl); } setOneSchema(className, schema) { - if (this.ttl <= 0) { + if (!this.ttl) { return Promise.resolve(null); } this.cache.put(SCHEMA_CACHE_PREFIX+className, schema, this.ttl); } getOneSchema(className) { - if (this.ttl <= 0) { + if (!this.ttl) { return Promise.resolve(null); } return this.cache.get(SCHEMA_CACHE_PREFIX+className); diff --git a/src/ParseServer.js b/src/ParseServer.js index de7cc75227..0ebad76441 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -140,7 +140,7 @@ class ParseServer { expireInactiveSessions = true, verbose = false, revokeSessionOnPasswordReset = true, - schemaCacheTTL = -1, // -1 = no cache + schemaCacheTTL = 0, // 0 = no cache __indexBuildCompletionCallbackForTests = () => {}, }) { // Initialize the node client SDK automatically diff --git a/src/cli/cli-definitions.js b/src/cli/cli-definitions.js index db34ee49e6..f0b7d0deee 100644 --- a/src/cli/cli-definitions.js +++ b/src/cli/cli-definitions.js @@ -200,7 +200,7 @@ export default { }, "schemaCacheTTL": { env: "PARSE_SERVER_SCHEMA_CACHE_TTL", - help: "The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to -1; disabled.", + help: "The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to 0; disabled.", action: numberParser("schemaCacheTTL"), } }; From 0a6eaf33d4a4b5d86b25db34f3e9ebb81e2049d2 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Tue, 19 Jul 2016 10:00:55 -0400 Subject: [PATCH 07/10] Use options object instead of boolean --- src/Controllers/DatabaseController.js | 4 +-- src/Controllers/SchemaController.js | 42 +++++++++++++-------------- src/Routers/SchemasRouter.js | 8 ++--- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 1adefbe835..61806d4450 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -109,9 +109,9 @@ DatabaseController.prototype.validateClassName = function(className) { }; // Returns a promise for a schemaController. -DatabaseController.prototype.loadSchema = function(force = false) { +DatabaseController.prototype.loadSchema = function(options = {clearCache: false}) { if (!this.schemaPromise) { - this.schemaPromise = SchemaController.load(this.adapter, this.schemaCache, force); + this.schemaPromise = SchemaController.load(this.adapter, this.schemaCache, options); this.schemaPromise.then(() => delete this.schemaPromise, () => delete this.schemaPromise); } diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index e68a199932..9e07eba351 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -282,13 +282,13 @@ class SchemaController { this.perms = {}; } - reloadData(clearCache = false) { + reloadData(options = {clearCache: false}) { this.data = {}; this.perms = {}; - if (clearCache) { + if (options.clearCache) { this._cache.clear(); } - return this.getAllClasses(clearCache) + return this.getAllClasses(options) .then(allSchemas => { allSchemas.forEach(schema => { this.data[schema.className] = injectDefaultSchema(schema).fields; @@ -306,12 +306,12 @@ class SchemaController { }); } - getAllClasses(clearCache = false) { - if (clearCache) { + getAllClasses(options = {clearCache: false}) { + if (options.clearCache) { this._cache.clear(); } return this._cache.getAllClasses().then((allClasses) => { - if (allClasses && allClasses.length && !clearCache) { + if (allClasses && allClasses.length && !options.clearCache) { return Promise.resolve(allClasses); } return this._dbAdapter.getAllClasses() @@ -323,17 +323,17 @@ class SchemaController { }); } - getOneSchema(className, allowVolatileClasses = false, clearCache) { - if (clearCache) { + getOneSchema(className, allowVolatileClasses = false, options = {clearCache: false}) { + if (options.clearCache) { this._cache.clear(); } + if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) { + return Promise.resolve(this.data[className]); + } return this._cache.getOneSchema(className).then((cached) => { - if (cached && !clearCache) { + if (cached && !options.clearCache) { return Promise.resolve(cached); } - if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) { - return Promise.resolve(this.data[className]); - } return this._dbAdapter.getClass(className) .then(injectDefaultSchema) .then((result) => { @@ -407,7 +407,7 @@ class SchemaController { }); return Promise.all(deletePromises) // Delete Everything - .then(() => this.reloadData(true)) // Reload our Schema, so we have all the new values + .then(() => this.reloadData({ clearCache: true })) // Reload our Schema, so we have all the new values .then(() => { let promises = insertedFields.map(fieldName => { const type = submittedFields[fieldName]; @@ -441,13 +441,13 @@ class SchemaController { // We don't have this class. Update the schema return this.addClassIfNotExists(className) // The schema update succeeded. Reload the schema - .then(() => this.reloadData(true)) + .then(() => this.reloadData({ clearCache: true })) .catch(error => { // The schema update failed. This can be okay - it might // have failed because there's a race condition and a different // client is making the exact same schema update that we want. // So just reload the schema. - return this.reloadData(true); + return this.reloadData({ clearCache: true }); }) .then(() => { // Ensure that the schema now validates @@ -517,7 +517,7 @@ class SchemaController { } validateCLP(perms, newSchema); return this._dbAdapter.setClassLevelPermissions(className, perms) - .then(() => this.reloadData(true)); + .then(() => this.reloadData({ clearCache: true })); } // Returns a promise that resolves successfully to the new schema @@ -557,14 +557,14 @@ class SchemaController { return this._dbAdapter.addFieldIfNotExists(className, fieldName, type).then(() => { // The update succeeded. Reload the schema - return this.reloadData(true); + return this.reloadData({ clearCache: true }); }, error => { //TODO: introspect the error and only reload if the error is one for which is makes sense to reload // The update failed. This can be okay - it might have been a race // condition where another client updated the schema in the same // way that we wanted to. So, just reload the schema - return this.reloadData(true); + return this.reloadData({ clearCache: true }); }).then(error => { // Ensure that the schema now validates if (!dbTypeMatchesObjectType(this.getExpectedType(className, fieldName), type)) { @@ -596,7 +596,7 @@ class SchemaController { throw new Parse.Error(136, `field ${fieldName} cannot be changed`); } - return this.getOneSchema(className, false, true) + return this.getOneSchema(className, false, {clearCache: true}) .catch(error => { if (error === undefined) { throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} does not exist.`); @@ -746,9 +746,9 @@ class SchemaController { } // Returns a promise for a new Schema. -const load = (dbAdapter, schemaCache, clearCache) => { +const load = (dbAdapter, schemaCache, options) => { let schema = new SchemaController(dbAdapter, schemaCache); - return schema.reloadData(clearCache).then(() => schema); + return schema.reloadData(options).then(() => schema); } // Builds a new schema (in schema API response format) out of an diff --git a/src/Routers/SchemasRouter.js b/src/Routers/SchemasRouter.js index 8a67a5af0c..71c40d55ee 100644 --- a/src/Routers/SchemasRouter.js +++ b/src/Routers/SchemasRouter.js @@ -15,14 +15,14 @@ function classNameMismatchResponse(bodyClass, pathClass) { } function getAllSchemas(req) { - return req.config.database.loadSchema(true) + return req.config.database.loadSchema({ clearCache: true}) .then(schemaController => schemaController.getAllClasses(true)) .then(schemas => ({ response: { results: schemas } })); } function getOneSchema(req) { const className = req.params.className; - return req.config.database.loadSchema(true) + return req.config.database.loadSchema({ clearCache: true}) .then(schemaController => schemaController.getOneSchema(className, true)) .then(schema => ({ response: schema })) .catch(error => { @@ -46,7 +46,7 @@ function createSchema(req) { throw new Parse.Error(135, `POST ${req.path} needs a class name.`); } - return req.config.database.loadSchema(true) + return req.config.database.loadSchema({ clearCache: true}) .then(schema => schema.addClassIfNotExists(className, req.body.fields, req.body.classLevelPermissions)) .then(schema => ({ response: schema })); } @@ -59,7 +59,7 @@ function modifySchema(req) { let submittedFields = req.body.fields || {}; let className = req.params.className; - return req.config.database.loadSchema(true) + return req.config.database.loadSchema({ clearCache: true}) .then(schema => schema.updateClass(className, submittedFields, req.body.classLevelPermissions, req.config.database)) .then(result => ({response: result})); } From eb59295c3c5a79d637233490a3d0f7d1ebd7ad63 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Wed, 20 Jul 2016 14:28:19 -0400 Subject: [PATCH 08/10] Imrpove concurrency of loadSchema --- src/Controllers/SchemaController.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 9e07eba351..fbea8e19f9 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -283,12 +283,15 @@ class SchemaController { } reloadData(options = {clearCache: false}) { - this.data = {}; - this.perms = {}; if (options.clearCache) { this._cache.clear(); } - return this.getAllClasses(options) + if (this.reloadDataPromise && !options.clearCache) { + return this.reloadDataPromise; + } + this.data = {}; + this.perms = {}; + this.reloadDataPromise = this.getAllClasses(options) .then(allSchemas => { allSchemas.forEach(schema => { this.data[schema.className] = injectDefaultSchema(schema).fields; @@ -303,7 +306,12 @@ class SchemaController { classLevelPermissions: {} }); }); + delete this.reloadDataPromise; + }, (err) => { + delete this.reloadDataPromise; + throw err; }); + return this.reloadDataPromise; } getAllClasses(options = {clearCache: false}) { From 5bb6c9989f97c59538a039928c55c78504f040e0 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Wed, 20 Jul 2016 16:06:10 -0400 Subject: [PATCH 09/10] Adds testing with SCHEMA_CACHE_ON --- .travis.yml | 1 + spec/helper.js | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index ee210a2a84..7bc82ec621 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ env: - MONGODB_VERSION=2.6.11 - MONGODB_VERSION=3.0.8 - MONGODB_VERSION=3.2.6 + - MONGODB_VERSION=3.2.6 TEST_SCHEMA_CACHE=3600 matrix: fast_finish: true, branches: diff --git a/spec/helper.js b/spec/helper.js index 8a1d0c5fd4..add51ec3e7 100644 --- a/spec/helper.js +++ b/spec/helper.js @@ -60,6 +60,7 @@ var defaultConfiguration = { module: path.resolve(__dirname, "myoauth") // relative path as it's run from src } }, + schemaCacheTTL: process.env.TEST_SCHEMA_CACHE || 0 }; let openConnections = {}; From 4e8c355bb18a480d9fe151b1be0b1074d6b8c325 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Fri, 22 Jul 2016 16:15:52 +0200 Subject: [PATCH 10/10] Use CacheController instead of generator - Makes caching SchemaCache use a generated prefix - Makes clearing the SchemaCache clear only the cached schema keys - Enable cache by default (ttl 5s) --- .travis.yml | 1 - spec/ParseInstallation.spec.js | 7 +++-- spec/RestQuery.spec.js | 10 +++++-- spec/Schema.spec.js | 4 +-- spec/helper.js | 3 +- src/Config.js | 5 +++- src/Controllers/DatabaseController.js | 1 - src/Controllers/SchemaCache.js | 40 +++++++++++++++++++++------ src/Controllers/SchemaController.js | 10 ++++--- src/ParseServer.js | 11 ++------ 10 files changed, 60 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7bc82ec621..ee210a2a84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ env: - MONGODB_VERSION=2.6.11 - MONGODB_VERSION=3.0.8 - MONGODB_VERSION=3.2.6 - - MONGODB_VERSION=3.2.6 TEST_SCHEMA_CACHE=3600 matrix: fast_finish: true, branches: diff --git a/spec/ParseInstallation.spec.js b/spec/ParseInstallation.spec.js index c1730985c0..ea12320a6e 100644 --- a/spec/ParseInstallation.spec.js +++ b/spec/ParseInstallation.spec.js @@ -9,8 +9,8 @@ let Parse = require('parse/node').Parse; let rest = require('../src/rest'); let request = require("request"); -let config = new Config('test'); -let database = config.database; +let config; +let database; let defaultColumns = require('../src/Controllers/SchemaController').defaultColumns; const installationSchema = { fields: Object.assign({}, defaultColumns._Default, defaultColumns._Installation) }; @@ -18,7 +18,8 @@ const installationSchema = { fields: Object.assign({}, defaultColumns._Default, describe('Installations', () => { beforeEach(() => { - config.database.schemaCache.clear(); + config = new Config('test'); + database = config.database; }); it_exclude_dbs(['postgres'])('creates an android installation with ids', (done) => { diff --git a/spec/RestQuery.spec.js b/spec/RestQuery.spec.js index 78d18d9a05..a6c8d9e7e1 100644 --- a/spec/RestQuery.spec.js +++ b/spec/RestQuery.spec.js @@ -9,11 +9,17 @@ var querystring = require('querystring'); var request = require('request'); var rp = require('request-promise'); -var config = new Config('test'); -let database = config.database; +var config; +let database; var nobody = auth.nobody(config); describe('rest query', () => { + + beforeEach(() => { + config = new Config('test'); + database = config.database; + }); + it('basic query', (done) => { rest.create(config, nobody, 'TestObject', {}).then(() => { return rest.find(config, nobody, 'TestObject', {}); diff --git a/spec/Schema.spec.js b/spec/Schema.spec.js index 1752cca671..2c6dc242b8 100644 --- a/spec/Schema.spec.js +++ b/spec/Schema.spec.js @@ -4,7 +4,7 @@ var Config = require('../src/Config'); var SchemaController = require('../src/Controllers/SchemaController'); var dd = require('deep-diff'); -var config = new Config('test'); +var config; var hasAllPODobject = () => { var obj = new Parse.Object('HasAllPOD'); @@ -21,7 +21,7 @@ var hasAllPODobject = () => { describe('SchemaController', () => { beforeEach(() => { - config.database.schemaCache.clear(); + config = new Config('test'); }); it('can validate one object', (done) => { diff --git a/spec/helper.js b/spec/helper.js index add51ec3e7..c724ea93b7 100644 --- a/spec/helper.js +++ b/spec/helper.js @@ -59,8 +59,7 @@ var defaultConfiguration = { myoauth: { module: path.resolve(__dirname, "myoauth") // relative path as it's run from src } - }, - schemaCacheTTL: process.env.TEST_SCHEMA_CACHE || 0 + } }; let openConnections = {}; diff --git a/src/Config.js b/src/Config.js index 89ea4cad82..c6cd30aab6 100644 --- a/src/Config.js +++ b/src/Config.js @@ -37,9 +37,12 @@ export class Config { // Create a new DatabaseController per request if (cacheInfo.databaseController) { - this.database = new DatabaseController(cacheInfo.databaseController.adapter, cacheInfo.createSchemaCache()); + const schemaCache = new SchemaCache(cacheInfo.cacheController, cacheInfo.schemaCacheTTL); + this.database = new DatabaseController(cacheInfo.databaseController.adapter, schemaCache); } + this.schemaCacheTTL = cacheInfo.schemaCacheTTL; + this.serverURL = cacheInfo.serverURL; this.publicServerURL = removeTrailingSlash(cacheInfo.publicServerURL); this.verifyUserEmails = cacheInfo.verifyUserEmails; diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index 61806d4450..f286907f13 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -8,7 +8,6 @@ var mongodb = require('mongodb'); var Parse = require('parse/node').Parse; var SchemaController = require('./SchemaController'); -import SchemaCache from './SchemaCache'; const deepcopy = require('deepcopy'); diff --git a/src/Controllers/SchemaCache.js b/src/Controllers/SchemaCache.js index f9dfa0b508..7a56f10710 100644 --- a/src/Controllers/SchemaCache.js +++ b/src/Controllers/SchemaCache.js @@ -1,44 +1,68 @@ const MAIN_SCHEMA = "__MAIN_SCHEMA"; const SCHEMA_CACHE_PREFIX = "__SCHEMA"; +const ALL_KEYS = "__ALL_KEYS"; + +import { randomString } from '../cryptoUtils'; export default class SchemaCache { cache: Object; - constructor(adapter, ttl) { + constructor(cacheController, ttl = 30) { this.ttl = ttl; - this.cache = adapter; -} + if (typeof ttl == 'string') { + this.ttl = parseInt(ttl); + } + this.cache = cacheController; + this.prefix = SCHEMA_CACHE_PREFIX+randomString(20); + } + + put(key, value) { + return this.cache.get(this.prefix+ALL_KEYS).then((allKeys) => { + allKeys = allKeys || {}; + allKeys[key] = true; + return Promise.all([this.cache.put(this.prefix+ALL_KEYS, allKeys, this.ttl), this.cache.put(key, value, this.ttl)]); + }); + } getAllClasses() { if (!this.ttl) { return Promise.resolve(null); } - return this.cache.get(SCHEMA_CACHE_PREFIX+MAIN_SCHEMA); + return this.cache.get(this.prefix+MAIN_SCHEMA); } setAllClasses(schema) { if (!this.ttl) { return Promise.resolve(null); } - this.cache.put(SCHEMA_CACHE_PREFIX+MAIN_SCHEMA, schema, this.ttl); + return this.put(this.prefix+MAIN_SCHEMA, schema); } setOneSchema(className, schema) { if (!this.ttl) { return Promise.resolve(null); } - this.cache.put(SCHEMA_CACHE_PREFIX+className, schema, this.ttl); + return this.put(this.prefix+className, schema); } getOneSchema(className) { if (!this.ttl) { return Promise.resolve(null); } - return this.cache.get(SCHEMA_CACHE_PREFIX+className); + return this.cache.get(this.prefix+className); } clear() { // That clears all caches... - this.cache.clear(); + let promise = Promise.resolve(); + return this.cache.get(this.prefix+ALL_KEYS).then((allKeys) => { + if (!allKeys) { + return; + } + let promises = Object.keys(allKeys).map((key) => { + return this.cache.del(key); + }); + return Promise.all(promises); + }); } } diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index fbea8e19f9..d321621f4f 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -325,8 +325,9 @@ class SchemaController { return this._dbAdapter.getAllClasses() .then(allSchemas => allSchemas.map(injectDefaultSchema)) .then(allSchemas => { - this._cache.setAllClasses(allSchemas); - return allSchemas; + return this._cache.setAllClasses(allSchemas).then(() => { + return allSchemas; + }); }) }); } @@ -345,8 +346,9 @@ class SchemaController { return this._dbAdapter.getClass(className) .then(injectDefaultSchema) .then((result) => { - this._cache.setOneSchema(className, result); - return result; + return this._cache.setOneSchema(className, result).then(() => { + return result; + }) }); }); } diff --git a/src/ParseServer.js b/src/ParseServer.js index 0ebad76441..be7d8d1ecd 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -140,7 +140,7 @@ class ParseServer { expireInactiveSessions = true, verbose = false, revokeSessionOnPasswordReset = true, - schemaCacheTTL = 0, // 0 = no cache + schemaCacheTTL = 5, // cache for 5s __indexBuildCompletionCallbackForTests = () => {}, }) { // Initialize the node client SDK automatically @@ -191,10 +191,6 @@ class ParseServer { const cacheControllerAdapter = loadAdapter(cacheAdapter, InMemoryCacheAdapter, {appId: appId}); const analyticsControllerAdapter = loadAdapter(analyticsAdapter, AnalyticsAdapter); - const createSchemaCache = function() { - let adapter = loadAdapter(cacheAdapter, InMemoryCacheAdapter, {appId: appId, ttl: schemaCacheTTL}); - return new SchemaCache(adapter, schemaCacheTTL); - } // We pass the options and the base class for the adatper, // Note that passing an instance would work too const filesController = new FilesController(filesControllerAdapter, appId); @@ -203,7 +199,7 @@ class ParseServer { const userController = new UserController(emailControllerAdapter, appId, { verifyUserEmails }); const liveQueryController = new LiveQueryController(liveQuery); const cacheController = new CacheController(cacheControllerAdapter, appId); - const databaseController = new DatabaseController(databaseAdapter, createSchemaCache()); + const databaseController = new DatabaseController(databaseAdapter, new SchemaCache(cacheController, schemaCacheTTL)); const hooksController = new HooksController(appId, databaseController, webhookKey); const analyticsController = new AnalyticsController(analyticsControllerAdapter); @@ -260,8 +256,7 @@ class ParseServer { jsonLogs, revokeSessionOnPasswordReset, databaseController, - schemaCacheTTL, - createSchemaCache + schemaCacheTTL }); // To maintain compatibility. TODO: Remove in some version that breaks backwards compatability