Merge pull request #128 from topcoder-platform/feature/data-scripts

Feature/data scripts

Author: maxceem

README.md

@@ -141,10 +141,15 @@
 | `npm run start`                                                                                                           | Start app in the production mode from prebuilt `dist` folder.        |
 | `npm run dev`                                                                                                             | Start app in the development mode using `nodemon`.                   |
 | `npm run test`                                                                                                            | Run tests.                                                           |
-| `npm run test-data`                                                                                                       | Clears and imports Data into ES.                                     |
 | `npm run init-db`                                                                                                         | Initializes Database.                                                |
-| `npm run create-index`                                                                                                    | Create Elasticsearch indexes.                                        |
-| `npm run delete-index`                                                                                                    | Delete Elasticsearch indexes.                                        |
+| `npm run create-index`                                                                                                    | Create Elasticsearch indexes. Use `-- --force` flag to skip confirmation                                       |
+| `npm run delete-index`                                                                                                    | Delete Elasticsearch indexes. Use `-- --force` flag to skip confirmation  |
+| `npm run data:import <filePath>`                                                                                                       | Imports data into ES and db from filePath (`./data/demo-data.json` is used as default). Use `-- --force` flag to skip confirmation |
+| `npm run data:export <filePath>`                                                                                                       | Exports data from ES and db into filePath (`./data/demo-data.json` is used as default). Use `-- --force` flag to skip confirmation |
+| `npm run index:all`                                                                                                       | Indexes all data from db into ES. Use `-- --force` flag to skip confirmation|
+| `npm run index:jobs <jobId>`                                                                                                       | Indexes job data from db into ES, if jobId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
+| `npm run index:job-candidates <jobCandidateId>`                                                                                                   | Indexes job candidate data from db into ES, if jobCandidateId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
+| `npm run index:resource-bookings <resourceBookingsId>`                                                                                                       | Indexes resource bookings data from db into ES, if resourceBookingsId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
 | `npm run services:up`                                                                                                     | Start services via docker-compose for local development.             |
 | `npm run services:down`                                                                                                   | Stop services via docker-compose for local development.              |
 | `npm run services:logs -- -f <service_name>`                                                                              | View logs of some service inside docker-compose.                     |

config/default.js

@@ -47,6 +47,9 @@ module.exports = {
   // the project service url
   PROJECT_API_URL: process.env.PROJECT_API_URL || 'https://api.topcoder-dev.com',
 
+  // the default path for importing and exporting data
+  DEFAULT_DATA_FILE_PATH: './data/demo-data.json',
+
   esConfig: {
     // the elasticsearch host
     HOST: process.env.ES_HOST || 'http://localhost:9200',
@@ -68,7 +71,12 @@ module.exports = {
     // the job candidate index
     ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate',
     // the resource booking index
-    ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking'
+    ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking',
+
+    // the max bulk size in MB for ES indexing
+    MAX_BULK_REQUEST_SIZE_MB: process.env.MAX_BULK_REQUEST_SIZE_MB || 20,
+    // the max number of documents per bulk for ES indexing
+    MAX_BULK_NUM_DOCUMENTS: process.env.MAX_BULK_NUM_DOCUMENTS || 100
   },
 
   // Topcoder Bus API URL