Merge pull request #272 from topcoder-platform/dev

[PROD] Next Release

Author: nkumar-topcoder

.circleci/config.yml

@@ -68,6 +68,7 @@ workflows:
             branches:
               only:
                 - dev
+                - change-validatations-in-job-jc
 
       # Production builds are exectuted only on tagged commits to the
       # master branch.

README.md

@@ -87,6 +87,9 @@
    tc-taas-es-processor | [2021-04-09T21:20:19.035Z] app INFO : Starting kafka consumer
    tc-taas-es-processor | 2021-04-09T21:20:21.292Z INFO no-kafka-client Joined group taas-es-processor generationId 1 as no-kafka-client-076538fc-60dd-4ca4-a2b9-520bdf73bc9e
    tc-taas-es-processor | 2021-04-09T21:20:21.293Z INFO no-kafka-client Elected as group leader
+   tc-taas-es-processor | 2021-04-09T21:20:21.449Z DEBUG no-kafka-client Subscribed to taas.role.update:0 offset 0 leader kafka:9093
+   tc-taas-es-processor | 2021-04-09T21:20:21.450Z DEBUG no-kafka-client Subscribed to taas.role.delete:0 offset 0 leader kafka:9093
+   tc-taas-es-processor | 2021-04-09T21:20:21.451Z DEBUG no-kafka-client Subscribed to taas.role.requested:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.452Z DEBUG no-kafka-client Subscribed to taas.jobcandidate.create:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.455Z DEBUG no-kafka-client Subscribed to taas.job.create:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.456Z DEBUG no-kafka-client Subscribed to taas.resourcebooking.delete:0 offset 0 leader kafka:9093
@@ -100,10 +103,11 @@
    tc-taas-es-processor | 2021-04-09T21:20:21.469Z DEBUG no-kafka-client Subscribed to taas.workperiodpayment.update:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.470Z DEBUG no-kafka-client Subscribed to taas.workperiodpayment.delete:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.471Z DEBUG no-kafka-client Subscribed to taas.workperiodpayment.create:0 offset 0 leader kafka:9093
+   tc-taas-es-processor | 2021-04-09T21:20:21.472Z DEBUG no-kafka-client Subscribed to taas.action.retry:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.473Z DEBUG no-kafka-client Subscribed to taas.job.update:0 offset 0 leader kafka:9093
    tc-taas-es-processor | 2021-04-09T21:20:21.474Z DEBUG no-kafka-client Subscribed to taas.resourcebooking.update:0 offset 0 leader kafka:9093
    tc-taas-es-processor | [2021-04-09T21:20:21.475Z] app INFO : Initialized.......
-   tc-taas-es-processor | [2021-04-09T21:20:21.479Z] app INFO : taas.job.create,taas.job.update,taas.job.delete,taas.jobcandidate.create,taas.jobcandidate.update,taas.jobcandidate.delete,taas.resourcebooking.create,taas.resourcebooking.update,taas.resourcebooking.delete,taas.workperiod.create,taas.workperiod.update,taas.workperiod.delete,taas.workperiodpayment.create,taas.workperiodpayment.update,taas.workperiodpayment.delete
+   tc-taas-es-processor | [2021-04-09T21:20:21.479Z] app INFO : common.error.reporting,taas.job.create,taas.job.update,taas.job.delete,taas.jobcandidate.create,taas.jobcandidate.update,taas.jobcandidate.delete,taas.resourcebooking.create,taas.resourcebooking.update,taas.resourcebooking.delete,taas.workperiod.create,taas.workperiod.update,taas.workperiod.delete,taas.workperiodpayment.create,taas.workperiodpayment.update,taas.interview.requested,taas.interview.update,taas.interview.bulkUpdate,taas.role.requested,taas.role.update,taas.role.delete,taas.action.retry
    tc-taas-es-processor | [2021-04-09T21:20:21.480Z] app INFO : Kick Start.......
    tc-taas-es-processor | ********** Topcoder Health Check DropIn listening on port 3001
    tc-taas-es-processor | Topcoder Health Check DropIn started and ready to roll
@@ -173,6 +177,19 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 2. Run `taas-es-processor` separately from the source code. As `npm run services:up` already run all the dependencies for both `taas-apis` and for `taas-es-processor`. The only thing you need to do for running `taas-es-processor` locally is clone the [taas-es-processor](https://github.com/topcoder-platform/taas-es-processor) repository and inside `taas-es-processor` folder run:
    - `nvm use` - to use correct Node version
    - `npm run install`
+   - Create `.env` file with the next environment variables. Values for **Auth0 config** should be shared with you on the forum.<br>
+
+      ```bash
+      # Auth0 config
+      AUTH0_URL=
+      AUTH0_AUDIENCE=
+      AUTH0_CLIENT_ID=
+      AUTH0_CLIENT_SECRET=
+      ```
+
+      - Values from this file would be automatically used by many `npm` commands.
+      - ⚠️ Never commit this file or its copy to the repository!
+
    - `npm run start`
 
 ## NPM Commands
@@ -194,7 +211,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 | `npm run index:jobs <jobId>`                                                                                              | Indexes job data from db into ES, if jobId is not given all data is indexed. Use `-- --force` flag to skip confirmation                            |
 | `npm run index:job-candidates <jobCandidateId>`                                                                           | Indexes job candidate data from db into ES, if jobCandidateId is not given all data is indexed. Use `-- --force` flag to skip confirmation         |
 | `npm run index:resource-bookings <resourceBookingsId>`                                                                    | Indexes resource bookings data from db into ES, if resourceBookingsId is not given all data is indexed. Use `-- --force` flag to skip confirmation |
-| `npm run index:work-periods <workPeriodId>`                                                                               | Indexes work periods data from db into ES, if workPeriodId is not given all data is indexed. Use `-- --force` flag to skip confirmation            |
+| `npm run index:roles <roleId>`                                                                                            | Indexes roles data from db into ES, if roleId is not given all data is indexed. Use `-- --force` flag to skip confirmation                         |
 | `npm run services:up`                                                                                                     | Start services via docker-compose for local development.                                                                                           |
 | `npm run services:down`                                                                                                   | Stop services via docker-compose for local development.                                                                                            |
 | `npm run services:logs -- -f <service_name>`                                                                              | View logs of some service inside docker-compose.                                                                                                   |
@@ -203,6 +220,7 @@ To be able to change and test `taas-es-processor` locally you can follow the nex
 | `npm run cov`                                                                                                             | Code Coverage Report.                                                                                                                              |
 | `npm run migrate`                                                                                                         | Run any migration files which haven't run yet.                                                                                                     |
 | `npm run migrate:undo`                                                                                                    | Revert most recent migration.                                                                                                                      |
+| `npm run demo-payment-scheduler`                                                                                          | Create 1000 Work Periods Payment records in with status "scheduled" and various "amount"                                                                         |
 
 ## Import and Export data
 

app-constants.js

@@ -34,6 +34,8 @@ const Scopes = {
   ALL_RESOURCE_BOOKING: 'all:taas-resourceBookings',
   // taas-team
   READ_TAAS_TEAM: 'read:taas-teams',
+  CREATE_ROLE_SEARCH_REQUEST: 'create:taas-roleSearchRequests',
+  CREATE_TAAS_TEAM: 'create:taas-teams',
   // work period
   READ_WORK_PERIOD: 'read:taas-workPeriods',
   CREATE_WORK_PERIOD: 'create:taas-workPeriods',
@@ -49,7 +51,13 @@ const Scopes = {
   READ_INTERVIEW: 'read:taas-interviews',
   CREATE_INTERVIEW: 'create:taas-interviews',
   UPDATE_INTERVIEW: 'update:taas-interviews',
-  ALL_INTERVIEW: 'all:taas-interviews'
+  ALL_INTERVIEW: 'all:taas-interviews',
+  // role
+  READ_ROLE: 'read:taas-roles',
+  CREATE_ROLE: 'create:taas-roles',
+  UPDATE_ROLE: 'update:taas-roles',
+  DELETE_ROLE: 'delete:taas-roles',
+  ALL_ROLE: 'all:taas-roles'
 }
 
 // Interview related constants
@@ -76,16 +84,85 @@ const ChallengeStatus = {
   COMPLETED: 'Completed'
 }
 
+/**
+ * Aggregate payment status for Work Period which is determined
+ * based on the payments the Work Period has using `PaymentStatusRules`
+ */
+const AggregatePaymentStatus = {
+  PENDING: 'pending',
+  IN_PROGRESS: 'in-progress',
+  PARTIALLY_COMPLETED: 'partially-completed',
+  COMPLETED: 'completed',
+  NO_DAYS: 'no-days'
+}
+
+/**
+ * `WorkPeriodPayment.status` - possible values
+ */
+const WorkPeriodPaymentStatus = {
+  COMPLETED: 'completed',
+  SCHEDULED: 'scheduled',
+  IN_PROGRESS: 'in-progress',
+  FAILED: 'failed',
+  CANCELLED: 'cancelled'
+}
+
+/**
+ * The rules how to determine WorkPeriod.paymentStatus based on the payments
+ *
+ * The top rule has priority over the bottom rules.
+ */
+const PaymentStatusRules = [
+  { paymentStatus: AggregatePaymentStatus.NO_DAYS, condition: { daysWorked: 0 } },
+  { paymentStatus: AggregatePaymentStatus.IN_PROGRESS, condition: { hasWorkPeriodPaymentStatus: [WorkPeriodPaymentStatus.SCHEDULED, WorkPeriodPaymentStatus.IN_PROGRESS] } },
+  { paymentStatus: AggregatePaymentStatus.COMPLETED, condition: { hasWorkPeriodPaymentStatus: [WorkPeriodPaymentStatus.COMPLETED], hasDueDays: false } },
+  { paymentStatus: AggregatePaymentStatus.PARTIALLY_COMPLETED, condition: { hasWorkPeriodPaymentStatus: [WorkPeriodPaymentStatus.COMPLETED], hasDueDays: true } },
+  { paymentStatus: AggregatePaymentStatus.PENDING, condition: { hasDueDays: true } }
+]
+
+/**
+ * The WorkPeriodPayment.status values which we take into account when calculate
+ * aggregate values inside WorkPeriod:
+ * - daysPaid
+ * - paymentTotal
+ * - paymentStatus
+ */
+const ActiveWorkPeriodPaymentStatuses = [
+  WorkPeriodPaymentStatus.SCHEDULED,
+  WorkPeriodPaymentStatus.IN_PROGRESS,
+  WorkPeriodPaymentStatus.COMPLETED
+]
+
+const WorkPeriodPaymentUpdateStatus = {
+  SCHEDULED: 'scheduled',
+  CANCELLED: 'cancelled'
+}
+
 const PaymentProcessingSwitch = {
   ON: 'ON',
   OFF: 'OFF'
 }
 
+const PaymentSchedulerStatus = {
+  START_PROCESS: 'start-process',
+  CREATE_CHALLENGE: 'create-challenge',
+  ASSIGN_MEMBER: 'assign-member',
+  ACTIVATE_CHALLENGE: 'activate-challenge',
+  GET_USER_ID: 'get-userId',
+  CLOSE_CHALLENGE: 'close-challenge'
+}
+
 module.exports = {
   UserRoles,
   FullManagePermissionRoles,
   Scopes,
   Interviews,
   ChallengeStatus,
-  PaymentProcessingSwitch
+  AggregatePaymentStatus,
+  WorkPeriodPaymentStatus,
+  WorkPeriodPaymentUpdateStatus,
+  PaymentSchedulerStatus,
+  PaymentProcessingSwitch,
+  PaymentStatusRules,
+  ActiveWorkPeriodPaymentStatuses
 }

app.js

@@ -13,6 +13,7 @@ const schedule = require('node-schedule')
 const logger = require('./src/common/logger')
 const eventHandlers = require('./src/eventHandlers')
 const interviewService = require('./src/services/InterviewService')
+const { processScheduler } = require('./src/services/PaymentSchedulerService')
 
 // setup express app
 const app = express()
@@ -97,6 +98,9 @@ const server = app.listen(app.get('port'), () => {
   eventHandlers.init()
   // schedule updateCompletedInterviews to run every hour
   schedule.scheduleJob('0 0 * * * *', interviewService.updateCompletedInterviews)
+
+  // schedule payment processing
+  schedule.scheduleJob(config.PAYMENT_PROCESSING.CRON, processScheduler)
 })
 
 if (process.env.NODE_ENV === 'test') {

config/default.js

@@ -40,7 +40,7 @@ module.exports = {
 
   TOPCODER_USERS_API: process.env.TOPCODER_USERS_API || 'https://api.topcoder-dev.com/v3/users',
   // the api to find topcoder members
-  TOPCODER_MEMBERS_API: process.env.TOPCODER_MEMBERS_API || 'https://api.topcoder-dev.com/v3/members',
+  TOPCODER_MEMBERS_API: process.env.TOPCODER_MEMBERS_API || 'https://api.topcoder-dev.com/v5/members',
   // rate limit of requests to user api
   MAX_PARALLEL_REQUEST_TOPCODER_USERS_API: process.env.MAX_PARALLEL_REQUEST_TOPCODER_USERS_API || 100,
 
@@ -76,8 +76,8 @@ module.exports = {
     ES_INDEX_JOB_CANDIDATE: process.env.ES_INDEX_JOB_CANDIDATE || 'job_candidate',
     // the resource booking index
     ES_INDEX_RESOURCE_BOOKING: process.env.ES_INDEX_RESOURCE_BOOKING || 'resource_booking',
-    // the work period index
-    ES_INDEX_WORK_PERIOD: process.env.ES_INDEX_WORK_PERIOD || 'work_period',
+    // the role index
+    ES_INDEX_ROLE: process.env.ES_INDEX_ROLE || 'role',
 
     // the max bulk size in MB for ES indexing
     MAX_BULK_REQUEST_SIZE_MB: process.env.MAX_BULK_REQUEST_SIZE_MB || 20,
@@ -133,6 +133,15 @@ module.exports = {
   TAAS_INTERVIEW_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_UPDATE_TOPIC || 'taas.interview.update',
   // the interview bulk update Kafka message topic
   TAAS_INTERVIEW_BULK_UPDATE_TOPIC: process.env.TAAS_INTERVIEW_BULK_UPDATE_TOPIC || 'taas.interview.bulkUpdate',
+  // topics for role service
+  // the create role entity Kafka message topic
+  TAAS_ROLE_CREATE_TOPIC: process.env.TAAS_ROLE_CREATE_TOPIC || 'taas.role.requested',
+  // the update role entity Kafka message topic
+  TAAS_ROLE_UPDATE_TOPIC: process.env.TAAS_ROLE_UPDATE_TOPIC || 'taas.role.update',
+  // the delete role entity Kafka message topic
+  TAAS_ROLE_DELETE_TOPIC: process.env.TAAS_ROLE_DELETE_TOPIC || 'taas.role.delete',
+  // special kafka topics
+  TAAS_ACTION_RETRY_TOPIC: process.env.TAAS_ACTION_RETRY_TOPIC || 'taas.action.retry',
 
   // the Kafka message topic for sending email
   EMAIL_TOPIC: process.env.EMAIL_TOPIC || 'external.action.email',
@@ -163,6 +172,45 @@ module.exports = {
   TYPE_ID_TASK: process.env.TYPE_ID_TASK || 'ecd58c69-238f-43a4-a4bb-d172719b9f31',
   DEFAULT_TIMELINE_TEMPLATE_ID: process.env.DEFAULT_TIMELINE_TEMPLATE_ID || '53a307ce-b4b3-4d6f-b9a1-3741a58f77e6',
   DEFAULT_TRACK_ID: process.env.DEFAULT_TRACK_ID || '9b6fc876-f4d9-4ccb-9dfd-419247628825',
-
-  PAYMENT_PROCESSING_SWITCH: process.env.PAYMENT_PROCESSING_SWITCH || 'OFF'
+  // the minimum matching rate when searching roles by skills
+  ROLE_MATCHING_RATE: process.env.ROLE_MATCHING_RATE || 0.70,
+  // member groups representing Wipro or TopCoder employee
+  INTERNAL_MEMBER_GROUPS: process.env.INTERNAL_MEMBER_GROUPS || ['20000000', '20000001', '20000003', '20000010', '20000015'],
+  // Topcoder skills cache time in minutes
+  TOPCODER_SKILLS_CACHE_TIME: process.env.TOPCODER_SKILLS_CACHE_TIME || 60,
+  // payment scheduler config
+  PAYMENT_PROCESSING: {
+    // switch off actual API calls in Payment Scheduler
+    SWITCH: process.env.PAYMENT_PROCESSING_SWITCH || 'OFF',
+    // the payment scheduler cron config
+    CRON: process.env.PAYMENT_PROCESSING_CRON || '0 */5 * * * *',
+    // the number of records processed by one time
+    BATCH_SIZE: parseInt(process.env.PAYMENT_PROCESSING_BATCH_SIZE || 50),
+    // in-progress expired to determine whether a record has been processed abnormally, moment duration format
+    IN_PROGRESS_EXPIRED: process.env.IN_PROGRESS_EXPIRED || 'PT1H',
+    // the number of max retry config
+    MAX_RETRY_COUNT: parseInt(process.env.PAYMENT_PROCESSING_MAX_RETRY_COUNT || 10),
+    // the time of retry base delay, unit: ms
+    RETRY_BASE_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_BASE_DELAY || 100),
+    // the time of retry max delay, unit: ms
+    RETRY_MAX_DELAY: parseInt(process.env.PAYMENT_PROCESSING_RETRY_MAX_DELAY || 10000),
+    // the max time of one request, unit: ms
+    PER_REQUEST_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_REQUEST_MAX_TIME || 30000),
+    // the max time of one payment record, unit: ms
+    PER_PAYMENT_MAX_TIME: parseInt(process.env.PAYMENT_PROCESSING_PER_PAYMENT_MAX_TIME || 60000),
+    // the max records of payment of a minute
+    PER_MINUTE_PAYMENT_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_PAYMENT_MAX_COUNT || 12),
+    // the max requests of challenge of a minute
+    PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 60),
+    // the max requests of resource of a minute
+    PER_MINUTE_RESOURCE_REQUEST_MAX_COUNT: parseInt(process.env.PAYMENT_PROCESSING_PER_MINUTE_CHALLENGE_REQUEST_MAX_COUNT || 20),
+    // the default step fix delay, unit: ms
+    FIX_DELAY_STEP: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of create challenge, unit: ms
+    FIX_DELAY_STEP_CREATE_CHALLENGE: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_CREATE_CHALLENGE || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of assign member, unit: ms
+    FIX_DELAY_STEP_ASSIGN_MEMBER: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_ASSIGN_MEMBER || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500),
+    // the fix delay after step of activate challenge, unit: ms
+    FIX_DELAY_STEP_ACTIVATE_CHALLENGE: parseInt(process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP_ACTIVATE_CHALLENGE || process.env.PAYMENT_PROCESSING_FIX_DELAY_STEP || 500)
+  }
 }