diff --git a/config/default.js b/config/default.js index f81211a..cc9b67e 100644 --- a/config/default.js +++ b/config/default.js @@ -35,5 +35,9 @@ module.exports = { ES_METADATA_INDEX: process.env.ES_METADATA_INDEX || 'metadata', ES_TYPE: process.env.ES_TYPE || 'doc', // ES 6.x accepts only 1 Type per index and it's mandatory to define it ES_METADATA_DEFAULT_ID: process.env.ES_METADATA_DEFAULT_ID || 1 // use for setting default id of metadata - } + }, + + // configuration for the stress test, see `test/stress/README.md` + STRESS_BASIC_QTY: process.env.STRESS_BASIC_QTY || 100, + STRESS_TESTER_TIMEOUT: process.env.STRESS_TESTER_TIMEOUT || 80 } diff --git a/package-lock.json b/package-lock.json index ab544cc..c5fe996 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3084,8 +3084,7 @@ "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", - "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==", - "optional": true + "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, "ms": { "version": "2.1.2", diff --git a/package.json b/package.json index 6745ef8..5b0fe5e 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ "sync:es": "node migrations/elasticsearch_sync.js", "view-data": "node test/common/view-data.js", "test": "NODE_ENV=test npm run sync:es && mocha test/e2e/*.test.js --timeout 30000 --exit && NODE_ENV=test npm run sync:es", - "test:cov": "nyc --reporter=html --reporter=text npm test" + "test:cov": "nyc --reporter=html --reporter=text npm test", + "test:stress": "NODE_ENV=test npm run sync:es && NODE_ENV=test node test/stress/doTest.js" }, "author": "TCSCODER", "license": "none", @@ -29,6 +30,7 @@ "http-aws-es": "^6.0.0", "joi": "^14.3.1", "lodash": "^4.17.11", + "moment": "^2.24.0", "no-kafka": "^3.4.3", "tc-core-library-js": "github:appirio-tech/tc-core-library-js#v2.6", "topcoder-healthcheck-dropin": "^1.0.3", diff --git a/src/services/ProcessorServiceMilestone.js b/src/services/ProcessorServiceMilestone.js index 9fa1d6c..128457a 100644 --- a/src/services/ProcessorServiceMilestone.js +++ b/src/services/ProcessorServiceMilestone.js @@ -88,14 +88,8 @@ async function create (message) { return _.assign(doc._source, { milestones }) } - // NOTE Disable indexing milestones when create at the moment, as it's now being indexed inside Project Service. - // It's because adding a milestones may cause cascading updates of other milestones and in such cases we are doing - // one ES index call instead of multiple calls. Otherwise ES may fail with error `version conflict`. - // This would be turned on back, as soon as we get rid of such cascading updates inside Project Service. - // - // await helper.updateTimelineESPromise(message.timelineId, updateDocPromise) - // logger.debug(`Milestone created successfully in elasticsearch index, (milestoneId: ${message.id})`) - logger.debug(`TEMPORARY SKIPPED: Milestone created successfully in elasticsearch index, (milestoneId: ${message.id})`) + await helper.updateTimelineESPromise(message.timelineId, updateDocPromise) + logger.debug(`Milestone created successfully in elasticsearch index, (milestoneId: ${message.id})`) } create.schema = { @@ -119,14 +113,8 @@ async function update (message) { return _.assign(doc._source, { milestones }) } - // NOTE Disable indexing milestones when update at the moment, as it's now being indexed inside Project Service. - // It's because updating a milestones may cause cascading updates of other milestones and in such cases we are doing - // one ES index call instead of multiple calls. Otherwise ES may fail with error `version conflict`. - // This would be turned on back, as soon as we get rid of such cascading updates inside Project Service. - // - // await helper.updateTimelineESPromise(message.timelineId, updateDocPromise) - // logger.debug(`Milestone updated successfully in elasticsearch index, (milestoneId: ${message.id})`) - logger.debug(`TEMPORARY SKIPPED: Milestone updated successfully in elasticsearch index, (milestoneId: ${message.id})`) + await helper.updateTimelineESPromise(message.timelineId, updateDocPromise) + logger.debug(`Milestone updated successfully in elasticsearch index, (milestoneId: ${message.id})`) } update.schema = { diff --git a/test/e2e/processor.timeline.index.test.js b/test/e2e/processor.timeline.index.test.js index 33473b5..f162aca 100644 --- a/test/e2e/processor.timeline.index.test.js +++ b/test/e2e/processor.timeline.index.test.js @@ -620,7 +620,7 @@ describe('TC Timeline And Nested Timeline Topic Tests', () => { }) }) -xdescribe('TC Milestone Topic Tests', () => { +describe('TC Milestone Topic Tests', () => { before(async () => { // runs before all tests in this block await ProcessorService.create(timelineCreatedMessage) diff --git a/test/stress/README.md b/test/stress/README.md new file mode 100644 index 0000000..6f5f188 --- /dev/null +++ b/test/stress/README.md @@ -0,0 +1,37 @@ +# Stress test + +## Overview + +This test is checking if any error would happen if we create, updated and delete milestones in **the same** timeline document in Elasticsearch `timelines` index at the same time. We are sending multiple Kafka events in parallel to trigger create, updated and delete milestone operations. + +1. First, the script should create initial data in the ES index: + - create one timeline in the `timeline` index with `2*STRESS_BASIC_QTY` milestones (it's important that all milestones belong to the same timeline) + +2. After initial data is created start an actual stress test by sending `3*STRESS_BASIC_QTY` Kafka messages in parallel: + - `STRESS_BASIC_QTY` Kafka messages to delete half of the initially created milestones + - `STRESS_BASIC_QTY` Kafka messages to update another half of initially created milestones + - `STRESS_BASIC_QTY` Kafka messages to create `STRESS_BASIC_QTY` new milestones in the same timeline + +## Configuration + +* `STRESS_BASIC_QTY`: The basic number of objects to use in stress test. + +* `STRESS_TESTER_TIMEOUT`: Number of seconds to wait after queueing create/update/delete requests and before validating data. Default is 80s, which is enough for `STRESS_BASIC_QTY=100`. This might have to be increased if `STRESS_BASIC_QTY` is higher than 100. + +## Run + +* Start processor + + It should point the **test** ES, so set `NODE_ENV=test`. + + ``` + NODE_ENV=test npm start + ``` + +* Run stress test + + It would test using **test** ES, as this command sets `NODE_ENV=test`. + + ``` + npm run test:stress + ``` diff --git a/test/stress/doTest.js b/test/stress/doTest.js new file mode 100644 index 0000000..5cb4720 --- /dev/null +++ b/test/stress/doTest.js @@ -0,0 +1,284 @@ +global.Promise = require('bluebird') +const config = require('config') +const Kafka = require('no-kafka') +const helper = require('../../src/common/helper') +const testHelper = require('../common/testHelper') +const _ = require('lodash') +const moment = require('moment') +const constants = require('../../src/constants') + +const producer = new Kafka.Producer({ + connectionString: config.KAFKA_URL, + handlerConcurrency: 1, + groupId: 'project-api' +}) + +const esClient = helper.getESClient() + +async function createTimeline () { + let now = moment().format() + + const timelinePayload = { + 'resource': constants.RESOURCES.TIMELINE, + 'createdAt': now, + 'updatedAt': now, + 'timestamp': now, + 'id': '1', + 'name': 'stress test timeline', + 'description': 'description', + 'startDate': now, + 'endDate': moment().add(7, 'days').format(), + 'reference': 'project', + 'referenceId': 1, + 'createdBy': 40051336, + 'updatedBy': 40051336, + 'milestones': [] + } + + const lastEndDate = moment().add(3, 'hours') + + _.forEach(_.range(1, config.STRESS_BASIC_QTY * 2 + 1), (i) => { + const milestonePayload = { + 'resource': constants.RESOURCES.MILESTONE, + 'timelineId': '1', + 'createdAt': now, + 'updatedAt': now, + 'startDate': lastEndDate.format(), + 'createdBy': 40051333, + 'updatedBy': 40051333, + 'hidden': false, + 'id': i, + 'name': 'original milestone ' + i, + 'description': 'description', + 'duration': 3, + 'completionDate': '2021-06-30T00:00:00.000Z', + 'status': 'open', + 'type': 'type3', + 'details': { + 'detail1': { + 'subDetail1C': 3 + }, + 'detail2': [ + 2, + 3, + 4 + ] + }, + 'order': 1, + 'plannedText': 'plannedText 3', + 'activeText': 'activeText 3', + 'completedText': 'completedText 3', + 'blockedText': 'blockedText 3', + 'actualStartDate': null + } + + lastEndDate.add(4, 'hours') + + milestonePayload.endDate = lastEndDate.format() + + timelinePayload.milestones.push(milestonePayload) + }) + + return esClient.create({ + index: config.get('esConfig.ES_TIMELINE_INDEX'), + type: config.get('esConfig.ES_TYPE'), + id: '1', + body: timelinePayload, + refresh: 'wait_for' + }) +} + +function shuffleArray (array) { + for (var i = array.length - 1; i > 0; i--) { + var j = Math.floor(Math.random() * (i + 1)) + var temp = array[i] + array[i] = array[j] + array[j] = temp + } + return array +} + +async function deleteMilestones (ids) { + const deletionRequests = _.map(ids, (i) => { + const payload = { + 'resource': constants.RESOURCES.MILESTONE, + 'timelineId': '1', + 'id': i + } + + const deleteMsg = { + 'topic': config.DELETE_DATA_TOPIC, + 'originator': 'project-api', + 'timestamp': moment().format(), + 'mime-type': 'application/json', + 'payload': payload + } + return producer.send({ + 'topic': config.DELETE_DATA_TOPIC, + 'message': { + 'value': JSON.stringify(deleteMsg) + } + }) + }) + return Promise.all(deletionRequests) +} + +async function updateMilestones (ids) { + const now = moment().format() + + const updateRequests = _.map(ids, (i) => { + const payload = { + 'resource': constants.RESOURCES.MILESTONE, + 'timelineId': '1', + 'id': i, + 'name': 'updated milestone ' + i, + 'duration': 4, + 'status': 'open', + 'type': 'type3', + 'order': 1, + 'plannedText': 'planned text 3', + 'activeText': 'active text 3', + 'completedText': ' completed text 3', + 'blockedText': 'blocked text 3' + } + const updateMsg = { + 'topic': config.UPDATE_DATA_TOPIC, + 'originator': 'project-api', + 'timestamp': now, + 'mime-type': 'application/json', + 'payload': payload + } + return producer.send({ + 'topic': config.UPDATE_DATA_TOPIC, + 'message': { + 'value': JSON.stringify(updateMsg) + } + }) + }) + + return Promise.all(updateRequests) +} + +async function createNewMilestones (ids) { + const now = moment().format() + const lastEndDate = moment().add(10, 'hours') + + const milestoneCreateRequests = _.map(ids, (i) => { + const milestonePayload = { + 'resource': constants.RESOURCES.MILESTONE, + 'timelineId': '1', + 'createdAt': now, + 'updatedAt': now, + 'startDate': lastEndDate.format(), + 'createdBy': 40051333, + 'updatedBy': 40051333, + 'hidden': false, + 'id': i, + 'name': 'new milestone ' + i, + 'description': 'description', + 'duration': 3, + 'completionDate': '2021-06-30T00:00:00.000Z', + 'status': 'open', + 'type': 'type3', + 'details': { + 'detail1': { + 'subDetail1C': 3 + }, + 'detail2': [ + 2, + 3, + 4 + ] + }, + 'order': 1, + 'plannedText': 'plannedText 3', + 'activeText': 'activeText 3', + 'completedText': 'completedText 3', + 'blockedText': 'blockedText 3', + 'actualStartDate': null + } + + lastEndDate.add(4, 'hours') + + milestonePayload.endDate = lastEndDate.format() + + const createMsg = { + 'topic': config.CREATE_DATA_TOPIC, + 'originator': 'project-api', + 'timestamp': moment().format(), + 'mime-type': 'application/json', + 'payload': milestonePayload + } + + return producer.send({ + 'topic': config.CREATE_DATA_TOPIC, + 'message': { + 'value': JSON.stringify(createMsg) + } + }) + }) + + return Promise.all(milestoneCreateRequests) +} + +async function sleep (n) { + return new Promise((resolve) => { + setTimeout(resolve, n) + }) +} + +async function main () { + await producer.init() + console.log('Creating initial data...') + await createTimeline() + console.log('Initial data is created.') + + const ids = shuffleArray(_.range(1, config.STRESS_BASIC_QTY * 2 + 1)) + const idsToDelete = ids.slice(0, config.STRESS_BASIC_QTY) + const idsToCreate = _.map(idsToDelete, (i) => i + 10000) + const idsToUpdate = ids.slice(config.STRESS_BASIC_QTY) + + console.log('Running multiple operations...') + await Promise.all([ + deleteMilestones(idsToDelete), + updateMilestones(idsToUpdate), + createNewMilestones(idsToCreate) + ]) + + console.log(`Waiting for ${config.STRESS_TESTER_TIMEOUT} seconds before validating the result data...`) + await sleep(1000 * config.STRESS_TESTER_TIMEOUT) + + const timeline = await testHelper.getTimelineESData('1') + + const milestones = {} + + _.forEach(timeline.milestones, (ms) => { + milestones[ms.id] = ms + }) + + _.forEach(idsToDelete, (i) => { + if (milestones[i]) { + console.error(`milestone with id: ${i} not deleted`) + } + }) + + _.forEach(idsToUpdate, (i) => { + if (!(milestones[i] && milestones[i].name === 'updated milestone ' + i)) { + console.error(`milestone with id: ${i} not updated`) + } + }) + + _.forEach(idsToCreate, (i) => { + if (!(milestones[i] && milestones[i].name === 'new milestone ' + i)) { + console.error(`milestone with id: ${i} not created`) + } + }) +} + +main().then(() => { + console.log('done') + process.exit(0) +}, (e) => { + console.log(e) + process.exit(1) +})