Optimized organization enrichment

backend/src/database/migrations/U1706788189__org-cache-enrichement.sql

@@ -0,0 +1,36 @@
+alter table "organizationCaches"
+    drop column if exists "affiliatedProfiles";
+alter table "organizationCaches"
+    drop column if exists "allSubsidiaries";
+alter table "organizationCaches"
+    drop column if exists "alternativeDomains";
+alter table "organizationCaches"
+    drop column if exists "alternativeNames";
+alter table "organizationCaches"
+    drop column if exists "averageEmployeeTenure";
+alter table "organizationCaches"
+    drop column if exists "averageTenureByLevel";
+alter table "organizationCaches"
+    drop column if exists "averageTenureByRole";
+alter table "organizationCaches"
+    drop column if exists "directSubsidiaries";
+alter table "organizationCaches"
+    drop column if exists "employeeChurnRate";
+alter table "organizationCaches"
+    drop column if exists "employeeCountByMonth";
+alter table "organizationCaches"
+    drop column if exists "employeeGrowthRate";
+alter table "organizationCaches"
+    drop column if exists "employeeCountByMonthByLevel";
+alter table "organizationCaches"
+    drop column if exists "employeeCountByMonthByRole";
+alter table "organizationCaches"
+    drop column if exists "gicsSector";
+alter table "organizationCaches"
+    drop column if exists "grossAdditionsByMonth";
+alter table "organizationCaches"
+    drop column if exists "grossDeparturesByMonth";
+alter table "organizationCaches"
+    drop column if exists "ultimateParent";
+alter table "organizationCaches"
+    drop column if exists "immediateParent";

backend/src/database/migrations/V1706788189__org-cache-enrichement.sql

@@ -0,0 +1,36 @@
+alter table "organizationCaches"
+    add column if not exists "affiliatedProfiles" text[];
+alter table "organizationCaches"
+    add column if not exists "allSubsidiaries" text[];
+alter table "organizationCaches"
+    add column if not exists "alternativeDomains" text[];
+alter table "organizationCaches"
+    add column if not exists "alternativeNames" text[];
+alter table "organizationCaches"
+    add column if not exists "averageEmployeeTenure" double precision;
+alter table "organizationCaches"
+    add column if not exists "averageTenureByLevel" jsonb;
+alter table "organizationCaches"
+    add column if not exists "averageTenureByRole" jsonb;
+alter table "organizationCaches"
+    add column if not exists "directSubsidiaries" text[];
+alter table "organizationCaches"
+    add column if not exists "employeeChurnRate" jsonb;
+alter table "organizationCaches"
+    add column if not exists "employeeCountByMonth" jsonb;
+alter table "organizationCaches"
+    add column if not exists "employeeGrowthRate" jsonb;
+alter table "organizationCaches"
+    add column if not exists "employeeCountByMonthByLevel" jsonb;
+alter table "organizationCaches"
+    add column if not exists "employeeCountByMonthByRole" jsonb;
+alter table "organizationCaches"
+    add column if not exists "gicsSector" text;
+alter table "organizationCaches"
+    add column if not exists "grossAdditionsByMonth" jsonb;
+alter table "organizationCaches"
+    add column if not exists "grossDeparturesByMonth" jsonb;
+alter table "organizationCaches"
+    add column if not exists "ultimateParent" text;
+alter table "organizationCaches"
+    add column if not exists "immediateParent" text;

backend/src/database/repositories/organizationCacheRepository.ts

@@ -31,41 +31,60 @@ class OrganizationCacheRepository {
     )
   }
 
+  private static COMMON_COLUMNS = [
+    'description',
+    'emails',
+    'phoneNumbers',
+    'logo',
+    'tags',
+    'twitter',
+    'linkedin',
+    'crunchbase',
+    'employees',
+    'revenueRange',
+    'importHash',
+    'location',
+    'github',
+    'lastEnrichedAt',
+    'employeeCountByCountry',
+    'type',
+    'geoLocation',
+    'size',
+    'ticker',
+    'headline',
+    'profiles',
+    'address',
+    'industry',
+    'founded',
+    'manuallyCreated',
+    'naics',
+    'affiliatedProfiles',
+    'allSubsidiaries',
+    'alternativeDomains',
+    'alternativeNames',
+    'averageEmployeeTenure',
+    'averageTenureByLevel',
+    'averageTenureByRole',
+    'directSubsidiaries',
+    'employeeChurnRate',
+    'employeeCountByMonth',
+    'employeeGrowthRate',
+    'employeeCountByMonthByLevel',
+    'employeeCountByMonthByRole',
+    'gicsSector',
+    'grossAdditionsByMonth',
+    'grossDeparturesByMonth',
+    'ultimateParent',
+    'immediateParent',
+  ]
+
   static async create(data, options: IRepositoryOptions) {
     const transaction = SequelizeRepository.getTransaction(options)
     const seq = SequelizeRepository.getSequelize(options)
 
     const record = await options.database.organizationCache.create(
       {
-        ...lodash.pick(data, [
-          'url',
-          'description',
-          'emails',
-          'phoneNumbers',
-          'logo',
-          'tags',
-          'twitter',
-          'linkedin',
-          'crunchbase',
-          'employees',
-          'revenueRange',
-          'importHash',
-          'enriched',
-          'github',
-          'location',
-          'employeeCountByCountry',
-          'type',
-          'ticker',
-          'headline',
-          'profiles',
-          'naics',
-          'industry',
-          'founded',
-          'address',
-          'size',
-          'lastEnrichedAt',
-          'manuallyCreated',
-        ]),
+        ...lodash.pick(data, ['url', 'enriched', ...OrganizationCacheRepository.COMMON_COLUMNS]),
       },
       {
         transaction,
@@ -113,38 +132,7 @@ class OrganizationCacheRepository {
 
     record = await record.update(
       {
-        ...lodash.pick(data, [
-          'url',
-          'description',
-          'emails',
-          'phoneNumbers',
-          'logo',
-          'tags',
-          'twitter',
-          'linkedin',
-          'crunchbase',
-          'employees',
-          'revenueRange',
-          'importHash',
-          'enriched',
-          'github',
-          'location',
-          'geoLocation',
-          'employeeCountByCountry',
-          'geoLocation',
-          'address',
-          'type',
-          'ticker',
-          'headline',
-          'profiles',
-          'naics',
-          'industry',
-          'founded',
-          'size',
-          'employees',
-          'twitter',
-          'lastEnrichedAt',
-        ]),
+        ...lodash.pick(data, ['url', 'enriched', ...OrganizationCacheRepository.COMMON_COLUMNS]),
       },
       {
         transaction,

services/apps/premium/organizations_enrichment_worker/src/activities.ts

@@ -1,19 +1,24 @@
 import {
-  getRemainingTenantCredits,
-  incrementTenantCredits,
-  getTenantOrganizationsForEnrichment,
-  tryEnrichOrganization,
+  getOrganizationCachesToEnrich,
+  tryEnrichOrganizationCache,
+  getMaxEnrichedOrganizationCachesPerExecution,
+} from './activities/enrichment'
+
+import {
   getApplicableTenants,
-  hasTenantOrganizationEnrichmentEnabled,
   syncToOpensearch,
-} from './activities/organizationEnrichment'
+  getRemainingTenantCredits,
+  incrementTenantCredits,
+  updateTenantOrganization,
+} from './activities/tenantUpdate'
 
 export {
   getApplicableTenants,
+  getOrganizationCachesToEnrich,
+  tryEnrichOrganizationCache,
+  getMaxEnrichedOrganizationCachesPerExecution,
+  syncToOpensearch,
   getRemainingTenantCredits,
   incrementTenantCredits,
-  getTenantOrganizationsForEnrichment,
-  tryEnrichOrganization,
-  hasTenantOrganizationEnrichmentEnabled,
-  syncToOpensearch,
+  updateTenantOrganization,
 }