Skip to content

Commit 5683896

Browse files
author
vikasrohit
authored
Merge pull request #284 from topcoder-platform/feature/query-improvements
Improve DB/ES queries
2 parents 4bf9163 + 1774ebe commit 5683896

File tree

5 files changed

+177
-64
lines changed

5 files changed

+177
-64
lines changed

‎migrations/elasticsearch_sync.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,39 @@ function getRequestBody(indexName) {
260260
},
261261
},
262262
},
263+
invites: {
264+
type: 'nested',
265+
properties: {
266+
createdAt: {
267+
type: 'date',
268+
format: 'strict_date_optional_time||epoch_millis',
269+
},
270+
createdBy: {
271+
type: 'integer',
272+
},
273+
email: {
274+
type: 'string',
275+
index: 'not_analyzed',
276+
},
277+
id: {
278+
type: 'long',
279+
},
280+
role: {
281+
type: 'string',
282+
index: 'not_analyzed',
283+
},
284+
updatedAt: {
285+
type: 'date',
286+
format: 'strict_date_optional_time||epoch_millis',
287+
},
288+
updatedBy: {
289+
type: 'integer',
290+
},
291+
userId: {
292+
type: 'long',
293+
},
294+
},
295+
},
263296
name: {
264297
type: 'string',
265298
},

‎src/models/project.js‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ module.exports = function defineProject(sequelize, DataTypes) {
9898
if (parameters.filters.id.$in.length === 0) {
9999
parameters.filters.id.$in.push(-1);
100100
}
101-
query += `AND id IN (${parameters.filters.id.$in}) `;
101+
query += `AND projects.id IN (${parameters.filters.id.$in}) `;
102102
} else if (_.isString(parameters.filters.id) || _.isNumber(parameters.filters.id)) {
103103
query += `AND id = ${parameters.filters.id} `;
104104
}
@@ -107,32 +107,51 @@ module.exports = function defineProject(sequelize, DataTypes) {
107107
const statusFilter = parameters.filters.status;
108108
if (_.isObject(statusFilter)) {
109109
const statuses = statusFilter.$in.join("','");
110-
query += `AND status IN ('${statuses}') `;
110+
query += `AND projects.status IN ('${statuses}') `;
111111
} else if (_.isString(statusFilter)) {
112-
query += `AND status ='${statusFilter}'`;
112+
query += `AND projects.status ='${statusFilter}'`;
113113
}
114114
}
115115
if (_.has(parameters.filters, 'type')) {
116-
query += `AND type = '${parameters.filters.type}' `;
116+
query += `AND projects.type = '${parameters.filters.type}' `;
117117
}
118118
if (_.has(parameters.filters, 'keyword')) {
119-
query += `AND "projectFullText" ~ lower('${parameters.filters.keyword}')`;
119+
query += `AND projects."projectFullText" ~ lower('${parameters.filters.keyword}')`;
120120
}
121121

122-
const attributesStr = `"${parameters.attributes.join('","')}"`;
122+
let joinQuery = '';
123+
if (_.has(parameters.filters, 'userId') || _.has(parameters.filters, 'email')) {
124+
query += ` AND (members."userId" = ${parameters.filters.userId}
125+
OR invites."userId" = ${parameters.filters.userId}
126+
OR invites."email" = '${parameters.filters.email}') GROUP BY projects.id`;
127+
128+
joinQuery = `LEFT OUTER JOIN project_members AS members ON projects.id = members."projectId"
129+
LEFT OUTER JOIN project_member_invites AS invites ON projects.id = invites."projectId"`;
130+
}
131+
132+
let attributesStr = _.map(parameters.attributes, attr => `projects."${attr}"`);
133+
attributesStr = `${attributesStr.join(',')}`;
123134
const orderStr = `"${parameters.order[0][0]}" ${parameters.order[0][1]}`;
124135

125136
// select count of projects
126-
return sequelize.query(`SELECT COUNT(1) FROM projects WHERE ${query}`,
137+
return sequelize.query(`SELECT COUNT(1) FROM projects AS projects
138+
${joinQuery}
139+
WHERE ${query}`,
127140
{ type: sequelize.QueryTypes.SELECT,
128141
logging: (str) => { log.debug(str); },
129142
raw: true,
130143
})
131144
.then((fcount) => {
132-
const count = fcount[0].count;
145+
let count = fcount.length;
146+
if (fcount.length === 1) {
147+
count = fcount[0].count;
148+
}
149+
133150
// select project attributes
134-
return sequelize.query(`SELECT ${attributesStr} FROM projects WHERE ${query} ORDER BY ` +
135-
` ${orderStr} LIMIT ${parameters.limit} OFFSET ${parameters.offset}`,
151+
return sequelize.query(`SELECT ${attributesStr} FROM projects AS projects
152+
${joinQuery}
153+
WHERE ${query} ORDER BY ` +
154+
` projects.${orderStr} LIMIT ${parameters.limit} OFFSET ${parameters.offset}`,
136155
{ type: sequelize.QueryTypes.SELECT,
137156
logging: (str) => { log.debug(str); },
138157
raw: true,

‎src/routes/admin/project-create-index.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,39 @@ function getRequestBody(indexName, docType) {
265265
},
266266
},
267267
},
268+
invites: {
269+
type: 'nested',
270+
properties: {
271+
createdAt: {
272+
type: 'date',
273+
format: 'strict_date_optional_time||epoch_millis',
274+
},
275+
createdBy: {
276+
type: 'integer',
277+
},
278+
email: {
279+
type: 'string',
280+
index: 'not_analyzed',
281+
},
282+
id: {
283+
type: 'long',
284+
},
285+
role: {
286+
type: 'string',
287+
index: 'not_analyzed',
288+
},
289+
updatedAt: {
290+
type: 'date',
291+
format: 'strict_date_optional_time||epoch_millis',
292+
},
293+
updatedBy: {
294+
type: 'integer',
295+
},
296+
userId: {
297+
type: 'long',
298+
},
299+
},
300+
},
268301
name: {
269302
type: 'string',
270303
},

‎src/routes/projects/list-db.js‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -133,30 +133,10 @@ module.exports = [
133133
}
134134

135135
// regular users can only see projects they are members of (or invited, handled bellow)
136-
const getProjectIds = models.ProjectMember.getProjectIdsForUser(req.authUser.userId);
137-
138-
return getProjectIds
139-
.then((accessibleProjectIds) => {
140-
let allowedProjectIds = accessibleProjectIds;
141-
// get projects with pending invite for current user
142-
const invites = models.ProjectMemberInvite.getProjectInvitesForUser(
143-
req.authUser.email,
144-
req.authUser.userId);
145-
if (invites) {
146-
allowedProjectIds = _.union(allowedProjectIds, invites);
147-
}
148-
// filter based on accessible
149-
if (_.get(criteria.filters, 'id', null)) {
150-
criteria.filters.id.$in = _.intersection(
151-
allowedProjectIds,
152-
criteria.filters.id.$in,
153-
);
154-
} else {
155-
criteria.filters.id = { $in: allowedProjectIds };
156-
}
157-
return retrieveProjects(req, criteria, sort, req.query.fields);
158-
})
159-
.then(result => res.json(util.wrapResponse(req.id, result.rows, result.count)))
160-
.catch(err => next(err));
136+
criteria.filters.userId = req.authUser.userId;
137+
criteria.filters.email = req.authUser.email;
138+
return retrieveProjects(req, criteria, sort, req.query.fields)
139+
.then(result => res.json(util.wrapResponse(req.id, result.rows, result.count)))
140+
.catch(err => next(err));
161141
},
162142
];

‎src/routes/projects/list.js‎

Lines changed: 77 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,56 @@ const buildEsFullTextQuery = (keyword, matchType, singleFieldName) => {
102102
};
103103
};
104104

105+
/**
106+
* Build ES query search request body based on userId and email
107+
*
108+
* @param {String} userId the user id
109+
* @param {String} email the email
110+
* @return {Array} query
111+
*/
112+
const buildEsShouldQuery = (userId, email) => {
113+
const should = [];
114+
if (userId) {
115+
should.push({
116+
nested: {
117+
path: 'members',
118+
query: {
119+
query_string: {
120+
query: userId,
121+
fields: ['members.userId'],
122+
},
123+
},
124+
},
125+
});
126+
should.push({
127+
nested: {
128+
path: 'invites',
129+
query: {
130+
query_string: {
131+
query: userId,
132+
fields: ['invites.userId'],
133+
},
134+
},
135+
},
136+
});
137+
}
138+
139+
if (email) {
140+
should.push({
141+
nested: {
142+
path: 'invites',
143+
query: {
144+
query_string: {
145+
query: email,
146+
fields: ['invites.email'],
147+
},
148+
},
149+
},
150+
});
151+
}
152+
return should;
153+
};
154+
105155
/**
106156
* Build ES query search request body based on value, keyword, matchType and fieldName
107157
*
@@ -234,6 +284,7 @@ const parseElasticSearchCriteria = (criteria, fields, order) => {
234284
// prepare the elasticsearch filter criteria
235285
const boolQuery = [];
236286
let mustQuery = [];
287+
let shouldQuery = [];
237288
let fullTextQuery;
238289
if (_.has(criteria, 'filters.id.$in')) {
239290
boolQuery.push({
@@ -269,6 +320,10 @@ const parseElasticSearchCriteria = (criteria, fields, order) => {
269320
['members.firstName', 'members.lastName']));
270321
}
271322

323+
if (_.has(criteria, 'filters.userId') || _.has(criteria, 'filters.email')) {
324+
shouldQuery = buildEsShouldQuery(criteria.filters.userId, criteria.filters.email);
325+
}
326+
272327
if (_.has(criteria, 'filters.status.$in')) {
273328
// status is an array
274329
boolQuery.push({
@@ -348,14 +403,29 @@ const parseElasticSearchCriteria = (criteria, fields, order) => {
348403
must: mustQuery,
349404
});
350405
}
406+
407+
if (shouldQuery.length > 0) {
408+
const newBody = { query: { bool: { must: [] } } };
409+
newBody.query.bool.must.push({
410+
bool: {
411+
should: shouldQuery,
412+
},
413+
});
414+
if (mustQuery.length > 0 || boolQuery.length > 0) {
415+
newBody.query.bool.must.push(body.query);
416+
}
417+
418+
body.query = newBody.query;
419+
}
420+
351421
if (fullTextQuery) {
352422
body.query = _.merge(body.query, fullTextQuery);
353423
if (body.query.bool) {
354424
body.query.bool.minimum_should_match = 1;
355425
}
356426
}
357427

358-
if (fullTextQuery || boolQuery.length > 0 || mustQuery.length > 0) {
428+
if (fullTextQuery || boolQuery.length > 0 || mustQuery.length > 0 || shouldQuery.length > 0) {
359429
searchCriteria.body = body;
360430
}
361431
return searchCriteria;
@@ -427,7 +497,6 @@ module.exports = [
427497
offset: req.query.offset || 0,
428498
};
429499
req.log.info(criteria);
430-
431500
if (!memberOnly
432501
&& (util.hasAdminRole(req)
433502
|| util.hasRoles(req, MANAGER_ROLES))) {
@@ -437,32 +506,11 @@ module.exports = [
437506
.catch(err => next(err));
438507
}
439508

440-
// regular users can only see projects they are members of (or invited, handled bellow)
441-
const getProjectIds = models.ProjectMember.getProjectIdsForUser(req.authUser.userId);
442-
443-
return getProjectIds
444-
.then((accessibleProjectIds) => {
445-
const allowedProjectIds = accessibleProjectIds;
446-
// get projects with pending invite for current user
447-
const invites = models.ProjectMemberInvite.getProjectInvitesForUser(
448-
req.authUser.email,
449-
req.authUser.userId);
450-
451-
return invites.then((ids => _.union(allowedProjectIds, ids)));
452-
})
453-
.then((allowedProjectIds) => {
454-
// filter based on accessible
455-
if (_.get(criteria.filters, 'id', null)) {
456-
criteria.filters.id.$in = _.intersection(
457-
allowedProjectIds,
458-
criteria.filters.id.$in,
459-
);
460-
} else {
461-
criteria.filters.id = { $in: allowedProjectIds };
462-
}
463-
return retrieveProjects(req, criteria, sort, req.query.fields);
464-
})
465-
.then(result => res.json(util.wrapResponse(req.id, result.rows, result.count)))
466-
.catch(err => next(err));
509+
// regular users can only see projects they are members of (or invited, handled below)
510+
criteria.filters.email = req.authUser.email;
511+
criteria.filters.userId = req.authUser.userId;
512+
return retrieveProjects(req, criteria, sort, req.query.fields)
513+
.then(result => res.json(util.wrapResponse(req.id, result.rows, result.count)))
514+
.catch(err => next(err));
467515
},
468516
];

0 commit comments

Comments
 (0)