Merge pull request #79 from eisbilir/work-period-automation

Work period automation

Author: maxceem

.circleci/config.yml

@@ -72,6 +72,7 @@ workflows:
               only:
                 - dev
                 - dev-circleci
+                - change-validatations-in-job-j
 
       # Production builds are exectuted only on tagged commits to the
       # master branch.

.gitignore

@@ -5,3 +5,4 @@ coverage
 .DS_Store
 .env
 api.env
+.eslintrc.y*ml

.nvmrc

@@ -0,0 +1 @@
+12

README.md

@@ -20,7 +20,9 @@ The following parameters can be set in config files or in env variables:
 - `KAFKA_CLIENT_CERT_KEY`: Kafka connection private key, optional;
     if not provided, then SSL connection is not used, direct insecure connection is used;
     if provided, it can be either path to private key file or private key content
+- `KAFKA_MESSAGE_ORIGINATOR`: The originator value for the kafka messages
 - `KAFKA_GROUP_ID`: the Kafka group id
+- `topics.KAFKA_ERROR_TOPIC`: the error topic at which bus api will publish any errors
 - `topics.TAAS_JOB_CREATE_TOPIC`: the create job entity Kafka message topic
 - `topics.TAAS_JOB_UPDATE_TOPIC`: the update job entity Kafka message topic
 - `topics.TAAS_JOB_DELETE_TOPIC`: the delete job entity Kafka message topic
@@ -33,6 +35,18 @@ The following parameters can be set in config files or in env variables:
 - `topics.TAAS_WORK_PERIOD_CREATE_TOPIC`: the create work period entity Kafka message topic
 - `topics.TAAS_WORK_PERIOD_UPDATE_TOPIC`: the update work period entity Kafka message topic
 - `topics.TAAS_WORK_PERIOD_DELETE_TOPIC`: the delete work period entity Kafka message topic
+- `topics.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC`: the create work period payment entity Kafka message topic
+- `topics.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC`: the update work period payment entity Kafka message topic
+- `topics.TAAS_INTERVIEW_REQUEST_TOPIC`: the request interview entity Kafka message topic
+- `topics.TAAS_INTERVIEW_UPDATE_TOPIC`: the update interview entity Kafka message topic
+- `topics.TAAS_INTERVIEW_BULK_UPDATE_TOPIC`: the bulk update interview entity Kafka message topic
+- `topics.TAAS_ROLE_CREATE_TOPIC`: the create role entity Kafka message topic
+- `topics.TAAS_ROLE_UPDATE_TOPIC`: the update role entity Kafka message topic
+- `topics.TAAS_ROLE_DELETE_TOPIC`: the delete role entity Kafka message topic
+- `topics.TAAS_ACTION_RETRY_TOPIC`: the retry process Kafka message topic
+- `MAX_RETRY`: maximum allowed retry count for failed operations for sending `taas.action.retry` message
+- `BASE_RETRY_DELAY`: base amount of retry delay (ms) for failed operations
+- `BUSAPI_URL`: Topcoder Bus API URL
 - `esConfig.HOST`: Elasticsearch host
 - `esConfig.AWS_REGION`: The Amazon region to use when using AWS Elasticsearch service
 - `esConfig.ELASTICCLOUD.id`: The elastic cloud id, if your elasticsearch instance is hosted on elastic cloud. DO NOT provide a value for ES_HOST if you are using this
@@ -41,13 +55,14 @@ The following parameters can be set in config files or in env variables:
 - `esConfig.ES_INDEX_JOB`: the index name for job
 - `esConfig.ES_INDEX_JOB_CANDIDATE`: the index name for job candidate
 - `esConfig.ES_INDEX_RESOURCE_BOOKING`: the index name for resource booking
-- `esConfig.ES_INDEX_WORK_PERIOD`: the index name for work period
+- `esConfig.ES_INDEX_ROLE`: the index name for role
 
 - `auth0.AUTH0_URL`: Auth0 URL, used to get TC M2M token
 - `auth0.AUTH0_AUDIENCE`: Auth0 audience, used to get TC M2M token
 - `auth0.AUTH0_CLIENT_ID`: Auth0 client id, used to get TC M2M token
 - `auth0.AUTH0_CLIENT_SECRET`: Auth0 client secret, used to get TC M2M token
 - `auth0.AUTH0_PROXY_SERVER_URL`: Proxy Auth0 URL, used to get TC M2M token
+- `auth0.TOKEN_CACHE_TIME`: Auth0 token cache time, used to get TC M2M token
 
 - `zapier.ZAPIER_COMPANYID_SLUG`: your company id in zapier; numeric value
 - `zapier.ZAPIER_CONTACTID_SLUG`: your contact id in zapier; numeric value
@@ -75,7 +90,13 @@ The following parameters can be set in config files or in env variables:
 
 ## Local deployment
 
-1. Make sure that Kafka and Elasticsearch is running as per instructions above.
+0. Make sure that Kafka and Elasticsearch is running as per instructions above.
+
+1. Make sure to use Node v12+ by command `node -v`. We recommend using [NVM](https://github.com/nvm-sh/nvm) to quickly switch to the right version:
+
+   ```bash
+   nvm use
+   ```
 
 2. From the project root directory, run the following command to install the dependencies
 
@@ -95,6 +116,21 @@ The following parameters can be set in config files or in env variables:
     npm run lint:fix
     ```
 
+4. Local config
+
+   In the `taas-es-processor` root directory create `.env` file with the next environment variables. Values for **Auth0 config** should be shared with you on the forum.<br>
+
+      ```bash
+      # Auth0 config
+      AUTH0_URL=
+      AUTH0_AUDIENCE=
+      AUTH0_CLIENT_ID=
+      AUTH0_CLIENT_SECRET=
+      ```
+
+      - Values from this file would be automatically used by many `npm` commands.
+      - ⚠️ Never commit this file or its copy to the repository!
+
 5. Start the processor and health check dropin
 
     ```bash

VERIFICATION.md

@@ -2,39 +2,51 @@
 
 ## Create documents in ES
 
-- Run the following commands to create `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES.
+- Run the following commands to create `Job`, `JobCandidate`, `Interview`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment`, `Role` documents in ES.
 
   ``` bash
   # for Job
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.job.create < test/messages/taas.job.create.event.json
   # for JobCandidate
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.jobcandidate.create < test/messages/taas.jobcandidate.create.event.json
+  # for Interview
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.interview.requested < test/messages/taas.interview.requested.event.json
   # for ResourceBooking
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.create < test/messages/taas.resourcebooking.create.event.json
   # for WorkPeriod
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.create < test/messages/taas.workperiod.create.event.json
+  # for WorkPeriodPayment
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiodpayment.create < test/messages/taas.workperiodpayment.create.event.json
+  # for Role
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.role.requested < test/messages/taas.role.create.event.json
   ```
 
 - Run `npm run view-data <model-name-here>` to see if documents were created.
 
 ## Update documents in ES
-- Run the following commands to update `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES.
+- Run the following commands to update `Job`, `JobCandidate`, `Interview`, `ResourceBooking`, `WorkPeriod`, `WorkPeriodPayment`, `Role` documents in ES.
 
   ``` bash
   # for Job
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.job.update < test/messages/taas.job.update.event.json
   # for JobCandidate
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.jobcandidate.update < test/messages/taas.jobcandidate.update.event.json
+  # for Interview
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.interview.update < test/messages/taas.interview.update.event.json
   # for ResourceBooking
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.update < test/messages/taas.resourcebooking.update.event.json
   # for WorkPeriod
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.update < test/messages/taas.workperiod.update.event.json
+  # for WorkPeriodPayment
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiodpayment.update < test/messages/taas.workperiodpayment.update.event.json
+  # for Role
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.role.update < test/messages/taas.role.update.event.json
   ```
 
 - Run `npm run view-data <model-name-here>` to see if documents were updated.
 
 ## Delete documents in ES
-- Run the following commands to delete `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod` documents in ES.
+- Run the following commands to delete `Job`, `JobCandidate`, `ResourceBooking`, `WorkPeriod`, `Role` documents in ES.
 
   ``` bash
   # for Job
@@ -45,6 +57,8 @@
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.resourcebooking.delete < test/messages/taas.resourcebooking.delete.event.json
   # for WorkPeriod
   docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.workperiod.delete < test/messages/taas.workperiod.delete.event.json
+  # for Role
+  docker exec -i taas-es-processor_kafka /opt/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic taas.role.delete < test/messages/taas.role.delete.event.json
   ```
 
 - Run `npm run view-data <model-name-here>` to see if documents were deleted.

config/default.js

@@ -1,7 +1,7 @@
 /**
  * The default configuration file.
  */
-
+require('dotenv').config()
 module.exports = {
   PORT: process.env.PORT || 3001,
   LOG_LEVEL: process.env.LOG_LEVEL || 'debug',
@@ -14,8 +14,12 @@ module.exports = {
 
   // Kafka group id
   KAFKA_GROUP_ID: process.env.KAFKA_GROUP_ID || 'taas-es-processor',
+  // The originator value for the kafka messages
+  KAFKA_MESSAGE_ORIGINATOR: process.env.KAFKA_MESSAGE_ORIGINATOR || 'taas-es-processor',
 
   topics: {
+    // The error topic at which bus api will publish any errors
+    KAFKA_ERROR_TOPIC: process.env.KAFKA_ERROR_TOPIC || 'common.error.reporting',
     // topics for job service
     TAAS_JOB_CREATE_TOPIC: process.env.TAAS_JOB_CREATE_TOPIC || 'taas.job.create',
     TAAS_JOB_UPDATE_TOPIC: process.env.TAAS_JOB_UPDATE_TOPIC || 'taas.job.update',
@@ -31,8 +35,28 @@ module.exports = {
     // topics for work period service
     TAAS_WORK_PERIOD_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_CREATE_TOPIC || 'taas.workperiod.create',
     TAAS_WORK_PERIOD_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_UPDATE_TOPIC || 'taas.workperiod.update',
-    TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete'
+    TAAS_WORK_PERIOD_DELETE_TOPIC: process.env.TAAS_WORK_PERIOD_DELETE_TOPIC || 'taas.workperiod.delete',
+    // topics for work period payment service
+    TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC: process.env.TAAS_WORK_PERIOD_PAYMENT_CREATE_TOPIC || 'taas.workperiodpayment.create',
+    TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC: process.env.TAAS_WORK_PERIOD_PAYMENT_UPDATE_TOPIC || 'taas.workperiodpayment.update',
+    // topics for interview service
+    TAAS_INTERVIEW_REQUEST_TOPIC: process.env.TAAS_INTERVIEW_REQUEST_TOPIC || 'taas.interview.requested',
+    TAAS_INTERVIEW_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_UPDATE_TOPIC || 'taas.interview.update',
+    TAAS_INTERVIEW_BULK_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_BULK_UPDATE_TOPIC || 'taas.interview.bulkUpdate',
+    // topics for role service
+    TAAS_ROLE_CREATE_TOPIC: process.env.TAAS_ROLE_CREATE_TOPIC || 'taas.role.requested',
+    TAAS_ROLE_UPDATE_TOPIC: process.env.TAAS_ROLE_UPDATE_TOPIC || 'taas.role.update',
+    TAAS_ROLE_DELETE_TOPIC: process.env.TAAS_ROLE_DELETE_TOPIC || 'taas.role.delete',
+    // special kafka topics
+    TAAS_ACTION_RETRY_TOPIC: process.env.TAAS_ACTION_RETRY_TOPIC || 'taas.action.retry'
+
   },
+  // maximum allowed retry count for failed operations for sending `action.retry` message
+  MAX_RETRY: process.env.MAX_RETRY || 3,
+  // base amount of retry delay for failed operations
+  BASE_RETRY_DELAY: process.env.BASE_RETRY_DELAY || 500,
+  // Topcoder Bus API URL
+  BUSAPI_URL: process.env.BUSAPI_URL || 'https://api.topcoder-dev.com/v5',
 
   esConfig: {
     HOST: process.env.ES_HOST || 'http://localhost:9200',
@@ -48,15 +72,16 @@ module.exports = {
     ES_INDEX_JOB: process.env.ES_INDEX_JOB || 'job',
     ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate',
     ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking',
-    ES_INDEX_WORK_PERIOD: process.env.ES_INDEX_WORK_PERIOD || 'work_period'
+    ES_INDEX_ROLE: process.env.ES_INDEX_ROLE || 'role'
   },
 
   auth0: {
     AUTH0_URL: process.env.AUTH0_URL,
     AUTH0_AUDIENCE: process.env.AUTH0_AUDIENCE,
     AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
     AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
-    AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL
+    AUTH0_PROXY_SERVER_URL: process.env.AUTH0_PROXY_SERVER_URL,
+    TOKEN_CACHE_TIME: process.env.TOKEN_CACHE_TIME
   },
 
   zapier: {

config/test.js

@@ -6,5 +6,7 @@ module.exports = {
   zapier: {
     ZAPIER_SWITCH: process.env.ZAPIER_SWITCH || 'ON',
     ZAPIER_JOB_CANDIDATE_SWITCH: process.env.ZAPIER_JOB_CANDIDATE_SWITCH || 'ON'
-  }
+  },
+  // don't retry actions during tests because tests for now don't expect it and should be updated first
+  MAX_RETRY: 0
 }

local/docker-compose.yml

@@ -12,7 +12,7 @@ services:
       - "9092:9092"
     environment:
       KAFKA_ADVERTISED_HOST_NAME: localhost
-      KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.resourcebooking.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.resourcebooking.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1"
+      KAFKA_CREATE_TOPICS: "taas.job.create:1:1,taas.jobcandidate.create:1:1,taas.interview.requested:1:1,taas.resourcebooking.create:1:1,taas.workperiod.create:1:1,taas.workperiodpayment.create:1:1,taas.job.update:1:1,taas.jobcandidate.update:1:1,taas.interview.update:1:1,taas.interview.bulkUpdate:1:1,taas.resourcebooking.update:1:1,taas.workperiod.update:1:1,taas.workperiodpayment.update:1:1,taas.job.delete:1:1,taas.jobcandidate.delete:1:1,taas.resourcebooking.delete:1:1,taas.workperiod.delete:1:1"
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
   esearch:
     image: elasticsearch:7.7.1