From 84631bd52ecb06a6dccfe3f3ea05a1f0f556da68 Mon Sep 17 00:00:00 2001 From: Abhimanyu Saharan Date: Sat, 13 May 2023 00:45:03 +0530 Subject: [PATCH 1/3] adds rq retry options #12327 --- docs/configuration/miscellaneous.md | 28 ++++++++++++++++++++++++++++ netbox/core/models/jobs.py | 4 +++- netbox/extras/webhooks.py | 4 +++- netbox/netbox/settings.py | 2 ++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index e3728acabef..cd34b3e510f 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -204,3 +204,31 @@ This parameter defines the URL of the repository that will be checked for new Ne Default: `300` The maximum execution time of a background task (such as running a custom script), in seconds. + +--- + +## RQ_RETRY_MAX + +!!! note + This parameter was added in NetBox v3.5. + +Default: `1` + +The maximum number of times a background task will be retried before being marked as failed. + +--- + +## RQ_RETRY_INTERVAL + +!!! note + This parameter was added in NetBox v3.5. + +Default: `[]` + +The number of seconds to wait before retrying a failed background task. This is a list of integers which will be used +as an interval. For example, `[60, 300, 3600]` will retry the task after 1 minute, 5 minutes, and 1 hour. + +!!! warning + If you use the interval, you should run your workers with `--with-scheduler` argument. + +--- diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 1d0eecd2195..3dfc82e5c3b 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -9,6 +9,7 @@ from django.urls import reverse from django.utils import timezone from django.utils.translation import gettext as _ +from rq import Retry from core.choices import JobStatusChoices from extras.constants import EVENT_JOB_END, EVENT_JOB_START @@ -219,5 +220,6 @@ def trigger_webhooks(self, event): event=event, data=self.data, timestamp=str(timezone.now()), - username=self.user.username + username=self.user.username, + retry=Retry(max=get_config().RQ_RETRY_MAX, interval=get_config().RQ_RETRY_INTERVAL) ) diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index 23702949a51..0049f4569e8 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -4,6 +4,7 @@ from django.contrib.contenttypes.models import ContentType from django.utils import timezone from django_rq import get_queue +from rq import Retry from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT @@ -116,5 +117,6 @@ def flush_webhooks(queue): snapshots=data['snapshots'], timestamp=str(timezone.now()), username=data['username'], - request_id=data['request_id'] + request_id=data['request_id'], + retry=Retry(max=get_config().RQ_RETRY_MAX, interval=get_config().RQ_RETRY_INTERVAL) ) diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 575755d2b5e..833eaf9bac6 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -140,6 +140,8 @@ REMOTE_AUTH_GROUP_SEPARATOR = getattr(configuration, 'REMOTE_AUTH_GROUP_SEPARATOR', '|') REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/') RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300) +RQ_RETRY_MAX = getattr(configuration, 'RQ_RETRY_MAX', 1) +RQ_RETRY_INTERVAL = getattr(configuration, 'RQ_RETRY_INTERVAL', []) SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/') SEARCH_BACKEND = getattr(configuration, 'SEARCH_BACKEND', 'netbox.search.backends.CachedValueSearchBackend') SECURE_SSL_REDIRECT = getattr(configuration, 'SECURE_SSL_REDIRECT', False) From 5c404cf604490e4281e695055f3771fee37ecc02 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 16 May 2023 11:34:17 -0400 Subject: [PATCH 2/3] Clean up docs; disable retries of failed jobs by default --- docs/configuration/miscellaneous.md | 18 ++++++------------ netbox/netbox/settings.py | 4 ++-- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index cd34b3e510f..728e61dcab1 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -207,28 +207,22 @@ The maximum execution time of a background task (such as running a custom script --- -## RQ_RETRY_MAX +## RQ_RETRY_INTERVAL !!! note This parameter was added in NetBox v3.5. -Default: `1` +Default: `60` -The maximum number of times a background task will be retried before being marked as failed. +This parameter controls how frequently a failed job is retried, up to the maximum number of times specified by `RQ_RETRY_MAX`. This must be either an integer specifying the number of seconds to wait between successive attempts, or a list of such values. For example, `[60, 300, 3600]` will retry the task after 1 minute, 5 minutes, and 1 hour. --- -## RQ_RETRY_INTERVAL +## RQ_RETRY_MAX !!! note This parameter was added in NetBox v3.5. -Default: `[]` +Default: `0` -The number of seconds to wait before retrying a failed background task. This is a list of integers which will be used -as an interval. For example, `[60, 300, 3600]` will retry the task after 1 minute, 5 minutes, and 1 hour. - -!!! warning - If you use the interval, you should run your workers with `--with-scheduler` argument. - ---- +The maximum number of times a background task will be retried before being marked as failed. diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 833eaf9bac6..ce5c4bee349 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -140,8 +140,8 @@ REMOTE_AUTH_GROUP_SEPARATOR = getattr(configuration, 'REMOTE_AUTH_GROUP_SEPARATOR', '|') REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/') RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300) -RQ_RETRY_MAX = getattr(configuration, 'RQ_RETRY_MAX', 1) -RQ_RETRY_INTERVAL = getattr(configuration, 'RQ_RETRY_INTERVAL', []) +RQ_RETRY_INTERVAL = getattr(configuration, 'RQ_RETRY_INTERVAL', 60) +RQ_RETRY_MAX = getattr(configuration, 'RQ_RETRY_MAX', 0) SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/') SEARCH_BACKEND = getattr(configuration, 'SEARCH_BACKEND', 'netbox.search.backends.CachedValueSearchBackend') SECURE_SSL_REDIRECT = getattr(configuration, 'SECURE_SSL_REDIRECT', False) From e3f5c806b86c24352bdc6a98cc308fbd6f891e57 Mon Sep 17 00:00:00 2001 From: jeremystretch Date: Tue, 16 May 2023 13:28:20 -0400 Subject: [PATCH 3/3] Pass a Retry object only if RQ_RETRY_MAX is non-zero --- docs/configuration/miscellaneous.md | 2 +- netbox/core/models/jobs.py | 5 ++--- netbox/extras/webhooks.py | 4 ++-- netbox/utilities/rqworker.py | 14 +++++++++++++- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/docs/configuration/miscellaneous.md b/docs/configuration/miscellaneous.md index 728e61dcab1..fd410a9d4ca 100644 --- a/docs/configuration/miscellaneous.md +++ b/docs/configuration/miscellaneous.md @@ -223,6 +223,6 @@ This parameter controls how frequently a failed job is retried, up to the maximu !!! note This parameter was added in NetBox v3.5. -Default: `0` +Default: `0` (retries disabled) The maximum number of times a background task will be retried before being marked as failed. diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index 3dfc82e5c3b..a91e75e611d 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -9,7 +9,6 @@ from django.urls import reverse from django.utils import timezone from django.utils.translation import gettext as _ -from rq import Retry from core.choices import JobStatusChoices from extras.constants import EVENT_JOB_END, EVENT_JOB_START @@ -17,7 +16,7 @@ from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from utilities.querysets import RestrictedQuerySet -from utilities.rqworker import get_queue_for_model +from utilities.rqworker import get_queue_for_model, get_rq_retry __all__ = ( 'Job', @@ -221,5 +220,5 @@ def trigger_webhooks(self, event): data=self.data, timestamp=str(timezone.now()), username=self.user.username, - retry=Retry(max=get_config().RQ_RETRY_MAX, interval=get_config().RQ_RETRY_INTERVAL) + retry=get_rq_retry() ) diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index 0049f4569e8..1fc869ee8e3 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -4,12 +4,12 @@ from django.contrib.contenttypes.models import ContentType from django.utils import timezone from django_rq import get_queue -from rq import Retry from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from netbox.registry import registry from utilities.api import get_serializer_for_model +from utilities.rqworker import get_rq_retry from utilities.utils import serialize_object from .choices import * from .models import Webhook @@ -118,5 +118,5 @@ def flush_webhooks(queue): timestamp=str(timezone.now()), username=data['username'], request_id=data['request_id'], - retry=Retry(max=get_config().RQ_RETRY_MAX, interval=get_config().RQ_RETRY_INTERVAL) + retry=get_rq_retry() ) diff --git a/netbox/utilities/rqworker.py b/netbox/utilities/rqworker.py index 5866dfee00c..61f5947670b 100644 --- a/netbox/utilities/rqworker.py +++ b/netbox/utilities/rqworker.py @@ -1,11 +1,12 @@ from django_rq.queues import get_connection -from rq import Worker +from rq import Retry, Worker from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT __all__ = ( 'get_queue_for_model', + 'get_rq_retry', 'get_workers_for_queue', ) @@ -22,3 +23,14 @@ def get_workers_for_queue(queue_name): Returns True if a worker process is currently servicing the specified queue. """ return Worker.count(get_connection(queue_name)) + + +def get_rq_retry(): + """ + If RQ_RETRY_MAX is defined and greater than zero, instantiate and return a Retry object to be + used when queuing a job. Otherwise, return None. + """ + retry_max = get_config().RQ_RETRY_MAX + retry_interval = get_config().RQ_RETRY_INTERVAL + if retry_max: + return Retry(max=retry_max, interval=retry_interval)