Merge pull request #545 from AliGebily/feature/export-import-aligebily-1

maxceem · web-flow · commit bfec5ad7fc17 · 2020-04-14T15:22:25.000+03:00
Topcoder Project Service - Import and Export Data - Final Fixes
diff --git a/package.json b/package.json
@@ -27,8 +27,8 @@
     "seed": "babel-node src/tests/seed.js --presets es2015",
     "demo-data": "babel-node local/seed",
     "es-db-compare": "babel-node scripts/es-db-compare",
-    "data:export": "babel-node scripts/data/export",
-    "data:import": "babel-node scripts/data/import"
+    "data:export": "LOG_LEVEL=info babel-node scripts/data/export",
+    "data:import": "LOG_LEVEL=info babel-node scripts/data/import"
   },
   "repository": {
     "type": "git",
diff --git a/scripts/data/export/exportData.js b/scripts/data/export/exportData.js
@@ -10,9 +10,9 @@ import { dataModels, validateDataModels } from '../dataModels';
  * @return {Promise}              Returns a promise
  */
 function saveExportedData(filePath, data, logger) {
-  logger.log('Start Saving data to file....');
+  logger.info('Start Saving data to file....');
   fs.writeFileSync(filePath, JSON.stringify(data));
-  logger.log('End Saving data to file....');
+  logger.info('End Saving data to file....');
 }
 /**
  * loads data  from database and export it to specified file path
@@ -38,7 +38,7 @@ async function exportDatabaseToJson(filePath, logger) {
     const modelName = dataModels[index];
     const modelRecords = results[index][0];
     allModelsRecords[modelName] = modelRecords;
-    logger.log(
+    logger.info(
       `Records loaded for model: ${modelName} = ${modelRecords.length}`,
     );
   }
diff --git a/scripts/data/export/index.js b/scripts/data/export/index.js
@@ -1,6 +1,7 @@
 import * as fs from 'fs';
 import * as path from 'path';
 import * as readline from 'readline';
+import Promise from 'bluebird';
 import util from '../../../src/util';
 import { exportData } from './exportData';
 /**
@@ -12,7 +13,7 @@ import { exportData } from './exportData';
 function runExportData(filePath, logger) {
   exportData(filePath, logger)
     .then(() => {
-      logger.log('Successfully exported data');
+      logger.info('Successfully exported data');
       process.exit(0);
     })
     .catch((err) => {
@@ -21,38 +22,39 @@ function runExportData(filePath, logger) {
     });
 }
 
-setTimeout(() => {
-  const logger = console;
-  const filePath =
-    process.argv[2] === '--file' && process.argv[3]
-      ? process.argv[3]
-      : 'data/demo-data.json';
-  logger.log('\nScript will export data to file:', filePath);
-  // check if file exists
-  if (fs.existsSync(filePath)) {
-    const rl = readline.createInterface({
-      input: process.stdin,
-      output: process.stdout,
-    });
-    // confirm overwritting to file
-    rl.question(
-      'File already exists, Are you sure to overwrite it? [Y] to overwrite: ',
-      (answer) => {
-        rl.close();
-        if (answer.toLowerCase() === 'y') {
-          logger.log('File will be overwritten.');
-          runExportData(filePath, logger);
-        } else {
-          logger.log('Exit without exporting any data');
-          process.exit(0);
-        }
-      },
-    ); // question()
-  } else {
-    // get base directory of the file
-    const baseDir = path.resolve(filePath, '..');
-    // create directory recursively if it does not exist
-    util.mkdirSyncRecursive(baseDir);
-    runExportData(filePath, logger);
-  }
+const logger = util.getAppLogger();
+const filePath =
+  process.argv[2] === '--file' && process.argv[3]
+    ? process.argv[3]
+    : 'data/demo-data.json';
+logger.info('Script will export data to file:', filePath);
+// check if file exists
+if (fs.existsSync(filePath)) {
+// We delay question for overwrite file, because the question overlaps with a warning message from sequelize module
+Promise.delay(1).then(() => {
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+  // confirm overwritting to file
+  rl.question(
+    'File already exists, Are you sure to overwrite it? [Y] to overwrite: ',
+    (answer) => {
+      rl.close();
+      if (answer.toLowerCase() === 'y') {
+        logger.info('File will be overwritten.');
+        runExportData(filePath, logger);
+      } else {
+        logger.info('Exit without exporting any data');
+        process.exit(0);
+      }
+    },
+  ); // question()
 });
+} else {
+  // get base directory of the file
+  const baseDir = path.resolve(filePath, '..');
+  // create directory recursively if it does not exist
+  util.mkdirSyncRecursive(baseDir);
+  runExportData(filePath, logger);
+}
diff --git a/scripts/data/import/importData.js b/scripts/data/import/importData.js
@@ -16,6 +16,7 @@ async function writeDataToDatabase(filePath, logger) {
     // Start a transaction
     transaction = await models.sequelize.transaction();
     const jsonData = JSON.parse(fs.readFileSync(filePath).toString());
+    // we disable no-await-in-loop because we need to run insert operations sequentially to avoid FK constraints errors
     /* eslint-disable no-await-in-loop */
     for (let index = 0; index < dataModels.length; index += 1) {
       const modelName = dataModels[index];
@@ -25,21 +26,21 @@ async function writeDataToDatabase(filePath, logger) {
         await models[modelName].bulkCreate(modelRecords, {
           transaction,
         });
-        logger.log(
+        logger.info(
           `Records to save for model: ${modelName} = ${modelRecords.length}`,
         );
       } else {
-        logger.log(`No records to save for model: ${modelName}`);
+        logger.info(`No records to save for model: ${modelName}`);
       }
     }
     // commit transaction only if all things went ok
-    logger.log('committing transaction to database...');
+    logger.info('committing transaction to database...');
     await transaction.commit();
   } catch (error) {
     logger.error('Error while writing data of model:', currentModelName);
     // rollback all insert operations
     if (transaction) {
-      logger.log('rollback database transaction...');
+      logger.info('rollback database transaction...');
       transaction.rollback();
     }
     if (error.name && error.errors && error.fields) {
@@ -65,10 +66,10 @@ async function writeDataToDatabase(filePath, logger) {
  * @return {Promise}              Returns a promise
  */
 async function indexDataToES(logger) {
-  logger.log('Indexing metatdata...');
+  logger.info('Indexing metatdata...');
   await indexMetadata();
 
-  logger.log('Indexing projects data...');
+  logger.info('Indexing projects data...');
   const req = {
     logger,
     projectIdStart: 1,
@@ -78,19 +79,7 @@ async function indexDataToES(logger) {
     fields: null,
     id: 0,
   };
-  await new Promise((resolve, reject) => {
-    indexProjectsRange(
-      req,
-      null,
-      (error) => {
-        if (error) {
-          reject(error);
-        } else {
-          resolve();
-        }
-      },
-    );
-  });
+  await indexProjectsRange(req);
 }
 
 /**
diff --git a/scripts/data/import/index.js b/scripts/data/import/index.js
@@ -1,17 +1,18 @@
 import * as fs from 'fs';
+import util from '../../../src/util';
 import { importData } from './importData';
 
-const logger = console;
+const logger = util.getAppLogger();
 const filePath = (process.argv[2] === '--file' && process.argv[3]) ? process.argv[3] : 'data/demo-data.json';
 // check if file exists
 if (!fs.existsSync(filePath)) {
   logger.error('File is not existing:', filePath);
   process.exit(1);
 } else {
-  logger.log('Script will import data from file:', filePath);
+  logger.info('Script will import data from file:', filePath);
   importData(filePath, logger)
     .then(() => {
-      logger.log('Successfully imported data');
+      logger.info('Successfully imported data');
       process.exit(0);
     })
     .catch((err) => {
diff --git a/src/middlewares/performanceRequestLogger.js b/src/middlewares/performanceRequestLogger.js
@@ -11,9 +11,7 @@ module.exports = function logRequest(logger) {
   }
 
   // Use the logger with memory usage info
-  return (request, response, next) => {
-    const req = request;
-    const res = response;
+  return (req, res, next) => {
     const startOpts = {
       method: req.method,
       url: req.url,
diff --git a/src/middlewares/userIdAuth.js b/src/middlewares/userIdAuth.js
@@ -9,13 +9,12 @@ const whitelistedOrigins = JSON.parse(config.get('whitelistedOriginsForUserIdAut
 
 /**
  * The userId authentication middleware.
- * @param {Object} request the request
+ * @param {Object} req the request
  * @param {Object} res the response
  * @param {Function} next the next middleware
  * @returns {Promise<void>} void
  */
-module.exports = function userIdAuth(request, res, next) { // eslint-disable-line consistent-return
-  const req = request;
+module.exports = function userIdAuth(req, res, next) { // eslint-disable-line consistent-return
   req.log.debug('Enter userIdAuth middleware');
 
   const bearerUserId = 'Bearer userId_';
diff --git a/src/middlewares/validateMilestoneTemplate.js b/src/middlewares/validateMilestoneTemplate.js
@@ -40,8 +40,7 @@ const validateMilestoneTemplate = {
    * @param {Function} next the express next middleware
    */
   // eslint-disable-next-line valid-jsdoc
-  validateRequestBody: (request, res, next) => {
-    const req = request;
+  validateRequestBody: (req, res, next) => {
     validateReference(req.body, req)
       .then(() => {
         if (req.body.sourceReference) {
@@ -110,13 +109,12 @@ const validateMilestoneTemplate = {
    * The middleware to validate milestoneTemplateId from request
    * path parameter, and set to the request params. This should be called after the validate()
    * middleware, and before the permissions() middleware.
-   * @param {Object} request the express request instance
+   * @param {Object} req the express request instance
    * @param {Object} res the express response instance
    * @param {Function} next the express next middleware
    */
   // eslint-disable-next-line valid-jsdoc
-  validateIdParam: (request, res, next) => {
-    const req = request;
+  validateIdParam: (req, res, next) => {
     models.MilestoneTemplate.findByPk(req.params.milestoneTemplateId)
       .then((milestoneTemplate) => {
         if (!milestoneTemplate) {
diff --git a/src/middlewares/validateTimeline.js b/src/middlewares/validateTimeline.js
@@ -7,12 +7,11 @@ import util from '../util';
 /**
  * Common validation code for types of timeline references.
  * @param {{ reference: string, referenceId: string|number }} sourceObject
- * @param {object} request
+ * @param {object} req
  * @param {boolean} [validateProjectExists]
  * @returns {Promise}
  */
-async function validateReference(sourceObject, request, validateProjectExists) {
-  const req = request;
+async function validateReference(sourceObject, req, validateProjectExists) {
   // The source object refers to a project
   if (sourceObject.reference === TIMELINE_REFERENCES.PROJECT) {
     // Set projectId to the params so it can be used in the permission check middleware
@@ -138,13 +137,12 @@ const validateTimeline = {
    * The middleware to validate and get the projectId specified by the timelineId from request
    * path parameter, and set to the request params. This should be called after the validate()
    * middleware, and before the permissions() middleware.
-   * @param {Object} request the express request instance
+   * @param {Object} req the express request instance
    * @param {Object} res the express response instance
    * @param {Function} next the express next middleware
    */
   // eslint-disable-next-line valid-jsdoc
-  validateTimelineIdParam: (request, res, next) => {
-    const req = request;
+  validateTimelineIdParam: (req, res, next) => {
     models.Timeline.findByPk(req.params.timelineId)
       .then((timeline) => {
         if (!timeline) {
diff --git a/src/permissions/connectManagerOrAdmin.ops.js b/src/permissions/connectManagerOrAdmin.ops.js
@@ -6,11 +6,10 @@ import models from '../models';
 
 /**
  * Only Connect Manager, Connect Admin, and administrator are allowed to perform the operations
- * @param {Object}    request         the express request instance
+ * @param {Object}    req         the express request instance
  * @return {Promise}              returns a promise
  */
-module.exports = request => new Promise(async (resolve, reject) => {
-  const req = request;
+module.exports = req => new Promise(async (resolve, reject) => {
   const hasAccess = util.hasRoles(req, MANAGER_ROLES);
 
   if (!hasAccess) {
diff --git a/src/permissions/copilotAndAbove.js b/src/permissions/copilotAndAbove.js
@@ -7,11 +7,10 @@ import { PERMISSION } from './constants';
  * Permission to allow copilot and above roles to perform certain operations
  * - User with Topcoder admins roles should be able to perform the operations.
  * - Project members with copilot and manager Project roles should be also able to perform the operations.
- * @param {Object}    request         the express request instance
+ * @param {Object}    req         the express request instance
  * @return {Promise}              returns a promise
  */
-module.exports = request => new Promise((resolve, reject) => {
-  const req = request;
+module.exports = req => new Promise((resolve, reject) => {
   const projectId = _.parseInt(req.params.projectId);
 
   return models.ProjectMember.getActiveProjectMembers(projectId)
diff --git a/src/permissions/project.anyAuthUser.js b/src/permissions/project.anyAuthUser.js
@@ -10,8 +10,7 @@
 import _ from 'lodash';
 import models from '../models';
 
-module.exports = (request) => {
-  const req = request;
+module.exports = (req) => {
   if (_.isUndefined(req.params.projectId)) {
     return Promise.reject(new Error('Policy "project.anyAuthUser" cannot be used for route without "projectId".'));
   }
diff --git a/src/routes/admin/project-index-create.js b/src/routes/admin/project-index-create.js
@@ -1,4 +1,3 @@
-
 import _ from 'lodash';
 import config from 'config';
 import { middleware as tcMiddleware } from 'tc-core-library-js';
@@ -32,24 +31,31 @@ module.exports = [
     const docType = _.get(req, 'body.docType', ES_PROJECT_TYPE);
     const fields = req.query.fields;
     const id = req.id;
-    indexProjectsRange(
-      {
-        logger,
-        projectIdStart,
-        projectIdEnd,
-        indexName,
-        docType,
-        fields,
-        id,
-      },
-      (esIndexingBody) => {
-        res.status(200).json({
-          message: `Reindex request successfully submitted for ${
-            esIndexingBody.length / 2
-          } projects`,
-        });
-      },
-      next,
-    );
+    return indexProjectsRange(
+        {
+          logger,
+          projectIdStart,
+          projectIdEnd,
+          indexName,
+          docType,
+          fields,
+          id,
+        },
+        (esIndexingBody) => {
+          res.status(200).json({
+            message: `Reindex request successfully submitted for ${
+              esIndexingBody.length / 2
+            } projects`,
+          });
+        },
+      ).then((result) => {
+        logger.debug(`project indexed successfully (projectId: ${projectIdStart}-${projectIdEnd})`, result);
+        logger.debug(result);
+      }).catch((error) => {
+        logger.error(
+          `Error in getting project details for indexing (projectId: ${projectIdStart}-${projectIdEnd})`,
+        error);
+        next(error);
+      });
   },
 ];
diff --git a/src/routes/index.js b/src/routes/index.js
diff --git a/src/routes/projectMemberInvites/update.js b/src/routes/projectMemberInvites/update.js
diff --git a/src/util.js b/src/util.js
diff --git a/src/utils/es.js b/src/utils/es.js
