From 7742c0d7d6329f367ff0a1dd7bd24e82ab789c64 Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Mon, 27 May 2024 13:37:32 +0530 Subject: [PATCH 1/6] Add default sync days var in env.example --- env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/env.example b/env.example index 27cbb1486..d9a8dea03 100644 --- a/env.example +++ b/env.example @@ -19,3 +19,4 @@ ANALYTICS_SERVER_PORT=9696 INTERNAL_API_BASE_URL=http://localhost:9696 INTERNAL_SYNC_API_BASE_URL=http://localhost:9697 NEXT_PUBLIC_APP_ENVIRONMENT="development" +DEFAULT_SYNC_DAYS=91 From b64e193e2c26284455aba8455aac7b9b7225f531 Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Mon, 27 May 2024 13:43:27 +0530 Subject: [PATCH 2/6] update default sync days to 90 in readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fc3274e49..7aef71d9c 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,7 @@ To set up middleware locally, follow these steps: REDIS_PORT=6385 ANALYTICS_SERVER_PORT=9696 SYNC_SERVER_PORT=9697 + DEFAULT_SYNC_DAYS=91 ``` - Start the backend servers From 641cc03ba22cead18f59b2cb68cf9ce437d69c9c Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Mon, 27 May 2024 13:44:10 +0530 Subject: [PATCH 3/6] remove unused const var DEFAULT_WORKFLOW_SYNC_DAYS --- .../mhq/service/workflows/sync/etl_github_actions_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/analytics_server/mhq/service/workflows/sync/etl_github_actions_handler.py b/backend/analytics_server/mhq/service/workflows/sync/etl_github_actions_handler.py index 02af62c52..bc0f1c2eb 100644 --- a/backend/analytics_server/mhq/service/workflows/sync/etl_github_actions_handler.py +++ b/backend/analytics_server/mhq/service/workflows/sync/etl_github_actions_handler.py @@ -20,7 +20,6 @@ from mhq.utils.log import LOG from mhq.utils.time import ISO_8601_DATE_FORMAT, time_now -DEFAULT_WORKFLOW_SYNC_DAYS = 31 WORKFLOW_PROCESSING_CHUNK_SIZE = 100 From 9ba46a88017f75a44681a9478eb43a6b92503970 Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Mon, 27 May 2024 13:45:22 +0530 Subject: [PATCH 4/6] Fetch default sync days from environment in etl handlers --- .../mhq/service/code/sync/etl_handler.py | 9 ++++++++- .../mhq/service/incidents/sync/etl_handler.py | 6 +++++- .../mhq/service/workflows/sync/etl_handler.py | 6 +++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/backend/analytics_server/mhq/service/code/sync/etl_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_handler.py index 7cf7b998b..667eb78ba 100644 --- a/backend/analytics_server/mhq/service/code/sync/etl_handler.py +++ b/backend/analytics_server/mhq/service/code/sync/etl_handler.py @@ -1,3 +1,4 @@ +from os import getenv from datetime import datetime, timedelta from typing import List @@ -19,6 +20,9 @@ class CodeETLHandler: + + DEFAULT_SYNC_DAYS = int(getenv("DEFAULT_SYNC_DAYS")) or 31 + def __init__( self, code_repo_service: CodeRepoService, @@ -92,7 +96,10 @@ def __sync_revert_prs_mapping( LOG.error(f"Error syncing revert PRs for repo {org_repo.name}: {str(e)}") raise e - def __get_org_repo_bookmark(self, org_repo: OrgRepo, default_sync_days: int = 31): + def __get_org_repo_bookmark( + self, org_repo: OrgRepo, default_sync_days: int = DEFAULT_SYNC_DAYS + ): + bookmark = self.code_repo_service.get_org_repo_bookmark( org_repo, BookmarkType.PR ) diff --git a/backend/analytics_server/mhq/service/incidents/sync/etl_handler.py b/backend/analytics_server/mhq/service/incidents/sync/etl_handler.py index 12695ca75..f5356d2df 100644 --- a/backend/analytics_server/mhq/service/incidents/sync/etl_handler.py +++ b/backend/analytics_server/mhq/service/incidents/sync/etl_handler.py @@ -1,3 +1,4 @@ +from os import getenv from datetime import timedelta from typing import List @@ -17,6 +18,9 @@ class IncidentsETLHandler: + + DEFAULT_SYNC_DAYS = int(getenv("DEFAULT_SYNC_DAYS")) or 31 + def __init__( self, provider: IncidentProvider, @@ -67,7 +71,7 @@ def _sync_service_incidents(self, service: OrgIncidentService): return def __get_incidents_bookmark( - self, service: OrgIncidentService, default_sync_days: int = 31 + self, service: OrgIncidentService, default_sync_days: int = DEFAULT_SYNC_DAYS ) -> IncidentsBookmark: bookmark = self.incident_repo_service.get_incidents_bookmark( str(service.id), IncidentBookmarkType.SERVICE, self.provider diff --git a/backend/analytics_server/mhq/service/workflows/sync/etl_handler.py b/backend/analytics_server/mhq/service/workflows/sync/etl_handler.py index db5a15695..1dd0c6f68 100644 --- a/backend/analytics_server/mhq/service/workflows/sync/etl_handler.py +++ b/backend/analytics_server/mhq/service/workflows/sync/etl_handler.py @@ -1,3 +1,4 @@ +from os import getenv from datetime import timedelta from typing import List, Tuple from uuid import uuid4 @@ -20,6 +21,9 @@ class WorkflowETLHandler: + + DEFAULT_SYNC_DAYS = int(getenv("DEFAULT_SYNC_DAYS")) or 31 + def __init__( self, code_repo_service: CodeRepoService, @@ -99,7 +103,7 @@ def _sync_repo_workflow(self, org_repo: OrgRepo, repo_workflow: RepoWorkflow): return def __get_repo_workflow_bookmark( - self, repo_workflow: RepoWorkflow, default_sync_days: int = 31 + self, repo_workflow: RepoWorkflow, default_sync_days: int = DEFAULT_SYNC_DAYS ) -> RepoWorkflowRunsBookmark: repo_workflow_bookmark = ( self.workflow_repo_service.get_repo_workflow_runs_bookmark(repo_workflow.id) From 10192436195508809fee86bec0f2ef06bcd54dd2 Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Mon, 27 May 2024 13:56:45 +0530 Subject: [PATCH 5/6] Update logic for default sync days across handlers --- backend/analytics_server/mhq/service/code/sync/etl_handler.py | 4 +++- .../mhq/service/incidents/sync/etl_handler.py | 4 +++- .../mhq/service/workflows/sync/etl_handler.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/analytics_server/mhq/service/code/sync/etl_handler.py b/backend/analytics_server/mhq/service/code/sync/etl_handler.py index 667eb78ba..01d9d5034 100644 --- a/backend/analytics_server/mhq/service/code/sync/etl_handler.py +++ b/backend/analytics_server/mhq/service/code/sync/etl_handler.py @@ -21,7 +21,9 @@ class CodeETLHandler: - DEFAULT_SYNC_DAYS = int(getenv("DEFAULT_SYNC_DAYS")) or 31 + DEFAULT_SYNC_DAYS = ( + int(getenv("DEFAULT_SYNC_DAYS")) if getenv("DEFAULT_SYNC_DAYS") else 31 + ) def __init__( self, diff --git a/backend/analytics_server/mhq/service/incidents/sync/etl_handler.py b/backend/analytics_server/mhq/service/incidents/sync/etl_handler.py index f5356d2df..5481e6cd8 100644 --- a/backend/analytics_server/mhq/service/incidents/sync/etl_handler.py +++ b/backend/analytics_server/mhq/service/incidents/sync/etl_handler.py @@ -19,7 +19,9 @@ class IncidentsETLHandler: - DEFAULT_SYNC_DAYS = int(getenv("DEFAULT_SYNC_DAYS")) or 31 + DEFAULT_SYNC_DAYS = ( + int(getenv("DEFAULT_SYNC_DAYS")) if getenv("DEFAULT_SYNC_DAYS") else 31 + ) def __init__( self, diff --git a/backend/analytics_server/mhq/service/workflows/sync/etl_handler.py b/backend/analytics_server/mhq/service/workflows/sync/etl_handler.py index 1dd0c6f68..a5c0cb995 100644 --- a/backend/analytics_server/mhq/service/workflows/sync/etl_handler.py +++ b/backend/analytics_server/mhq/service/workflows/sync/etl_handler.py @@ -22,7 +22,9 @@ class WorkflowETLHandler: - DEFAULT_SYNC_DAYS = int(getenv("DEFAULT_SYNC_DAYS")) or 31 + DEFAULT_SYNC_DAYS = ( + int(getenv("DEFAULT_SYNC_DAYS")) if getenv("DEFAULT_SYNC_DAYS") else 31 + ) def __init__( self, From 817ea36e47666840e83541f7ddc71263a8d64fdb Mon Sep 17 00:00:00 2001 From: Samad Yar Khan Date: Mon, 27 May 2024 15:02:16 +0530 Subject: [PATCH 6/6] Update defaults day to 31 --- README.md | 2 +- env.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7aef71d9c..4eaa1c5ed 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ To set up middleware locally, follow these steps: REDIS_PORT=6385 ANALYTICS_SERVER_PORT=9696 SYNC_SERVER_PORT=9697 - DEFAULT_SYNC_DAYS=91 + DEFAULT_SYNC_DAYS=31 ``` - Start the backend servers diff --git a/env.example b/env.example index d9a8dea03..e73a20759 100644 --- a/env.example +++ b/env.example @@ -19,4 +19,4 @@ ANALYTICS_SERVER_PORT=9696 INTERNAL_API_BASE_URL=http://localhost:9696 INTERNAL_SYNC_API_BASE_URL=http://localhost:9697 NEXT_PUBLIC_APP_ENVIRONMENT="development" -DEFAULT_SYNC_DAYS=91 +DEFAULT_SYNC_DAYS=31