chore: import/export scripts improvements

recreated demo data with valid projects data and dummified prices in metadata

Author: Maksym Mykhailenko

README.md

@@ -154,7 +154,7 @@ Local setup should work good on **Linux** and **macOS**. But **Windows** is not
 4. ♻ Init DB, ES and demo data (it clears any existent data)
 
    ```bash
-   npm run local:setup
+   npm run local:init
    ```
 
    This command will do 3 things:
@@ -215,7 +215,7 @@ npm run data:import -- --file path/to-file.json
 
 - As this commands calls topcoder services to get data like members details, so you have to provide environment variables `AUTH0_CLIENT_ID`, `AUTH0_CLIENT_SECRET`, `AUTH0_URL`, `AUTH0_AUDIENCE`, `AUTH0_PROXY_SERVER_URL`, they would automatically picked up from the `.env` file if provided.
 
-- If you encounter conflicts errors during import, you may need to clear database using `NODE_ENV=development  npm run sync:db`, and clear ES (Elasticsearch) indices using `NODE_ENV=development npm run sync:es`.
+- If you encounter conflicts errors during import, you may need to recreated database tables and Elasticssearch indexes by `npm run sync:all`.
 
 - List of models that will be imported are defined in `scripts/data/dataModels.js`. You can add new models to this list, but make sure that new models are added to list such that each model comes after its dependencies.
 

data/demo-data.json

@@ -10,6 +10,7 @@
     "lint": "./node_modules/.bin/eslint .",
     "lint:fix": "./node_modules/.bin/eslint . --fix || true",
     "build": "babel src -d dist --presets es2015 --copy-files",
+    "sync:all": "NODE_ENV=development npm run sync:db && NODE_ENV=development npm run sync:es",
     "sync:db": "./node_modules/.bin/babel-node migrations/sync.js",
     "sync:es": "./node_modules/.bin/babel-node migrations/elasticsearch_sync.js",
     "sync:es:metadata": "./node_modules/.bin/babel-node migrations/elasticsearch_sync.js --index-name metadata",
@@ -26,10 +27,10 @@
     "test:watch": "NODE_ENV=test ./node_modules/.bin/mocha -w --require babel-core/register $(find src -path '*spec.js*')",
     "demo-data": "babel-node local/seed",
     "es-db-compare": "babel-node scripts/es-db-compare",
-    "data:export": "LOG_LEVEL=info node --require dotenv/config --require babel-core/register scripts/data/export | bunyan -o short",
-    "data:import": "LOG_LEVEL=info node --require dotenv/config --require babel-core/register scripts/data/import | bunyan -o short",
+    "data:export": "NODE_ENV=development LOG_LEVEL=info node --require dotenv/config --require babel-core/register scripts/data/export",
+    "data:import": "NODE_ENV=development LOG_LEVEL=info node --require dotenv/config --require babel-core/register scripts/data/import",
     "local:run-docker": "docker-compose -f ./local/full/docker-compose.yml up -d",
-    "local:setup": "NODE_ENV=development npm run sync:db && NODE_ENV=development npm run sync:es && npm run data:import"
+    "local:init": "npm run sync:all && npm run data:import"
   },
   "repository": {
     "type": "git",

package.json

@@ -22,7 +22,7 @@ function runExportData(filePath, logger) {
     });
 }
 
-const logger = util.getAppLogger();
+const logger = util.getScriptsLogger();
 const filePath =
   process.argv[2] === '--file' && process.argv[3]
     ? process.argv[3]

scripts/data/export/index.js

@@ -61,13 +61,13 @@ async function writeDataToDatabase(filePath, logger) {
                 transaction,
               })
             )[0][0].setval;
-            logger.info(
+            logger.debug(
               `Updated autoIncrement for ${modelName}.${field} with max value = ${setValue}`,
             );
           }
         }
       } else {
-        logger.info(`No records to save for model: ${modelName}`);
+        logger.info(`No records to import for model: ${modelName}`);
       }
     }
     // commit transaction only if all things went ok

scripts/data/import/importData.js

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import util from '../../../src/util';
 import { importData } from './importData';
 
-const logger = util.getAppLogger();
+const logger = util.getScriptsLogger();
 const filePath = (process.argv[2] === '--file' && process.argv[3]) ? process.argv[3] : 'data/demo-data.json';
 // check if file exists
 if (!fs.existsSync(filePath)) {

scripts/data/import/index.js

@@ -1350,7 +1350,8 @@ _.assignIn(util, {
       return curDir;
     }, initDir);
   },
-  getAppLogger: () => {
+
+  getScriptsLogger: () => {
     const appName = 'tc-projects-service';
     return coreLib.logger({
       name: appName,