diff --git a/docs/configuration/required-parameters.md b/docs/configuration/required-parameters.md index 012d8576268..bda36599525 100644 --- a/docs/configuration/required-parameters.md +++ b/docs/configuration/required-parameters.md @@ -59,10 +59,7 @@ DATABASE = { ## REDIS -[Redis](https://redis.io/) is an in-memory data store similar to memcached. While Redis has been an optional component of -NetBox since the introduction of webhooks in version 2.4, it is required starting in 2.6 to support NetBox's caching -functionality (as well as other planned features). In 2.7, the connection settings were broken down into two sections for -task queuing and caching, allowing the user to connect to different Redis instances/databases per feature. +[Redis](https://redis.io/) is a lightweight in-memory data store similar to memcached. NetBox employs Redis for background task queuing and other features. Redis is configured using a configuration setting similar to `DATABASE` and these settings are the same for both of the `tasks` and `caching` subsections: @@ -81,7 +78,7 @@ REDIS = { 'tasks': { 'HOST': 'redis.example.com', 'PORT': 1234, - 'USERNAME': 'netbox' + 'USERNAME': 'netbox', 'PASSWORD': 'foobar', 'DATABASE': 0, 'SSL': False, @@ -89,7 +86,7 @@ REDIS = { 'caching': { 'HOST': 'localhost', 'PORT': 6379, - 'USERNAME': '' + 'USERNAME': '', 'PASSWORD': '', 'DATABASE': 1, 'SSL': False, diff --git a/docs/development/application-registry.md b/docs/development/application-registry.md index c845cd5a7f7..570563431bf 100644 --- a/docs/development/application-registry.md +++ b/docs/development/application-registry.md @@ -31,7 +31,7 @@ A dictionary of particular features (e.g. custom fields) mapped to the NetBox mo 'dcim': ['site', 'rack', 'devicetype', ...], ... }, - 'webhooks': { + 'event_rules': { 'extras': ['configcontext', 'tag', ...], 'dcim': ['site', 'rack', 'devicetype', ...], }, diff --git a/docs/development/models.md b/docs/development/models.md index d4838570a47..f04610ad564 100644 --- a/docs/development/models.md +++ b/docs/development/models.md @@ -10,19 +10,19 @@ The Django [content types](https://docs.djangoproject.com/en/stable/ref/contrib/ Depending on its classification, each NetBox model may support various features which enhance its operation. Each feature is enabled by inheriting from its designated mixin class, and some features also make use of the [application registry](./application-registry.md#model_features). -| Feature | Feature Mixin | Registry Key | Description | -|------------------------------------------------------------|-------------------------|--------------------|--------------------------------------------------------------------------------| -| [Change logging](../features/change-logging.md) | `ChangeLoggingMixin` | - | Changes to these objects are automatically recorded in the change log | -| Cloning | `CloningMixin` | - | Provides the `clone()` method to prepare a copy | -| [Custom fields](../customization/custom-fields.md) | `CustomFieldsMixin` | `custom_fields` | These models support the addition of user-defined fields | -| [Custom links](../customization/custom-links.md) | `CustomLinksMixin` | `custom_links` | These models support the assignment of custom links | -| [Custom validation](../customization/custom-validation.md) | `CustomValidationMixin` | - | Supports the enforcement of custom validation rules | -| [Export templates](../customization/export-templates.md) | `ExportTemplatesMixin` | `export_templates` | Users can create custom export templates for these models | -| [Job results](../features/background-jobs.md) | `JobsMixin` | `jobs` | Users can create custom export templates for these models | -| [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | -| [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | -| [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | -| [Webhooks](../integrations/webhooks.md) | `WebhooksMixin` | `webhooks` | NetBox is capable of generating outgoing webhooks for these objects | +| Feature | Feature Mixin | Registry Key | Description | +|------------------------------------------------------------|-------------------------|--------------------|-----------------------------------------------------------------------------------------| +| [Change logging](../features/change-logging.md) | `ChangeLoggingMixin` | - | Changes to these objects are automatically recorded in the change log | +| Cloning | `CloningMixin` | - | Provides the `clone()` method to prepare a copy | +| [Custom fields](../customization/custom-fields.md) | `CustomFieldsMixin` | `custom_fields` | These models support the addition of user-defined fields | +| [Custom links](../customization/custom-links.md) | `CustomLinksMixin` | `custom_links` | These models support the assignment of custom links | +| [Custom validation](../customization/custom-validation.md) | `CustomValidationMixin` | - | Supports the enforcement of custom validation rules | +| [Export templates](../customization/export-templates.md) | `ExportTemplatesMixin` | `export_templates` | Users can create custom export templates for these models | +| [Job results](../features/background-jobs.md) | `JobsMixin` | `jobs` | Users can create custom export templates for these models | +| [Journaling](../features/journaling.md) | `JournalingMixin` | `journaling` | These models support persistent historical commentary | +| [Synchronized data](../integrations/synchronized-data.md) | `SyncedDataMixin` | `synced_data` | Certain model data can be automatically synchronized from a remote data source | +| [Tagging](../models/extras/tag.md) | `TagsMixin` | `tags` | The models can be tagged with user-defined tags | +| [Event rules](../features/event-rules.md) | `EventRulesMixin` | `event_rules` | Event rules can send webhooks or run custom scripts automatically in response to events | ## Models Index @@ -111,7 +111,7 @@ Component models represent individual physical or virtual components belonging t ### Component Template Models -These function as templates to effect the replication of device and virtual machine components. Component template models support a limited feature set, including change logging, custom validation, and webhooks. +These function as templates to effect the replication of device and virtual machine components. Component template models support a limited feature set, including change logging, custom validation, and event rules. * [dcim.ConsolePortTemplate](../models/dcim/consoleporttemplate.md) * [dcim.ConsoleServerPortTemplate](../models/dcim/consoleserverporttemplate.md) diff --git a/docs/features/api-integration.md b/docs/features/api-integration.md index 8c0843bfea6..94a39d73173 100644 --- a/docs/features/api-integration.md +++ b/docs/features/api-integration.md @@ -26,9 +26,9 @@ To learn more about this feature, check out the [GraphQL API documentation](../i ## Webhooks -A webhook is a mechanism for conveying to some external system a change that took place in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. This can be done by creating a webhook for the device model in NetBox and identifying the webhook receiver. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. Webhooks are an excellent mechanism for building event-based automation processes. +A webhook is a mechanism for conveying to some external system a change that has taken place in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. To do this, first create a [webhook](../models/extras/webhook.md) identifying the remote receiver (URL), HTTP method, and any other necessary parameters. Then, define an [event rule](../models/extras/eventrule.md) which is triggered by device changes to transmit the webhook. -To learn more about this feature, check out the [webhooks documentation](../integrations/webhooks.md). +When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. Webhooks are an excellent mechanism for building event-based automation processes. To learn more about this feature, check out the [webhooks documentation](../integrations/webhooks.md). ## Prometheus Metrics diff --git a/docs/features/event-rules.md b/docs/features/event-rules.md new file mode 100644 index 00000000000..0e953522379 --- /dev/null +++ b/docs/features/event-rules.md @@ -0,0 +1,31 @@ +# Event Rules + +NetBox includes the ability to execute certain functions in response to internal object changes. These include: + +* [Scripts](../customization/custom-scripts.md) execution +* [Webhooks](../integrations/webhooks.md) execution + +For example, suppose you want to automatically configure a monitoring system to start monitoring a device when its operational status is changed to active, and remove it from monitoring for any other status. You can create a webhook in NetBox for the device model and craft its content and destination URL to effect the desired change on the receiving system. You can then associate an event rule with this webhook and the webhook will be sent automatically by NetBox whenever the configured constraints are met. + +Each event must be associated with at least one NetBox object type and at least one event (e.g. create, update, or delete). + +## Conditional Event Rules + +An event rule may include a set of conditional logic expressed in JSON used to control whether an event triggers for a specific object. For example, you may wish to trigger an event for devices only when the `status` field of an object is "active": + +```json +{ + "and": [ + { + "attr": "status.value", + "value": "active" + } + ] +} +``` + +For more detail, see the reference documentation for NetBox's [conditional logic](../reference/conditions.md). + +## Event Rule Processing + +When a change is detected, any resulting events are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing event(s) to be processed. The events are then extracted from the queue by the `rqworker` process. The current event queue and any failed events can be inspected in the admin UI under System > Background Tasks. diff --git a/docs/index.md b/docs/index.md index 05cd79f2353..84334337b24 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,7 +32,7 @@ In addition to its expansive and robust data model, NetBox offers myriad mechani * Custom fields * Custom model validation * Export templates -* Webhooks +* Event rules * Plugins * REST & GraphQL APIs diff --git a/docs/integrations/webhooks.md b/docs/integrations/webhooks.md index 9a1094988d1..8913fd99c16 100644 --- a/docs/integrations/webhooks.md +++ b/docs/integrations/webhooks.md @@ -1,11 +1,9 @@ # Webhooks -NetBox can be configured to transmit outgoing webhooks to remote systems in response to internal object changes. The receiver can act on the data in these webhook messages to perform related tasks. +NetBox can be configured via [Event Rules](../features/event-rules.md) to transmit outgoing webhooks to remote systems in response to internal object changes. The receiver can act on the data in these webhook messages to perform related tasks. For example, suppose you want to automatically configure a monitoring system to start monitoring a device when its operational status is changed to active, and remove it from monitoring for any other status. You can create a webhook in NetBox for the device model and craft its content and destination URL to effect the desired change on the receiving system. Webhooks will be sent automatically by NetBox whenever the configured constraints are met. -Each webhook must be associated with at least one NetBox object type and at least one event (create, update, or delete). Users can specify the receiver URL, HTTP request type (`GET`, `POST`, etc.), content type, and headers. A request body can also be specified; if left blank, this will default to a serialized representation of the affected object. - !!! warning "Security Notice" Webhooks support the inclusion of user-submitted code to generate the URL, custom headers, and payloads, which may pose security risks under certain conditions. Only grant permission to create or modify webhooks to trusted users. @@ -70,26 +68,12 @@ If no body template is specified, the request body will be populated with a JSON } ``` -## Conditional Webhooks - -A webhook may include a set of conditional logic expressed in JSON used to control whether a webhook triggers for a specific object. For example, you may wish to trigger a webhook for devices only when the `status` field of an object is "active": - -```json -{ - "and": [ - { - "attr": "status.value", - "value": "active" - } - ] -} -``` - -For more detail, see the reference documentation for NetBox's [conditional logic](../reference/conditions.md). +!!! note + The setting of conditional webhooks has been moved to [Event Rules](../features/event-rules.md) since NetBox 3.7 ## Webhook Processing -When a change is detected, any resulting webhooks are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing webhook(s) to be processed. The webhooks are then extracted from the queue by the `rqworker` process and HTTP requests are sent to their respective destinations. The current webhook queue and any failed webhooks can be inspected in the admin UI under System > Background Tasks. +Using [Event Rules](../features/event-rules.md), when a change is detected, any resulting webhooks are placed into a Redis queue for processing. This allows the user's request to complete without needing to wait for the outgoing webhook(s) to be processed. The webhooks are then extracted from the queue by the `rqworker` process and HTTP requests are sent to their respective destinations. The current webhook queue and any failed webhooks can be inspected in the admin UI under System > Background Tasks. A request is considered successful if the response has a 2XX status code; otherwise, the request is marked as having failed. Failed requests may be retried manually via the admin UI. diff --git a/docs/models/extras/eventrule.md b/docs/models/extras/eventrule.md new file mode 100644 index 00000000000..89645be3cff --- /dev/null +++ b/docs/models/extras/eventrule.md @@ -0,0 +1,35 @@ +# EventRule + +An event rule is a mechanism for automatically taking an action (such as running a script or sending a webhook) in response to an event in NetBox. For example, you may want to notify a monitoring system whenever the status of a device is updated in NetBox. This can be done by creating an event for device objects and designating a webhook to be transmitted. When NetBox detects a change to a device, an HTTP request containing the details of the change and who made it be sent to the specified receiver. + +See the [event rules documentation](../features/event-rules.md) for more information. + +## Fields + +### Name + +A unique human-friendly name. + +### Content Types + +The type(s) of object in NetBox that will trigger the rule. + +### Enabled + +If not selected, the event rule will not be processed. + +### Events + +The events which will trigger the rule. At least one event type must be selected. + +| Name | Description | +|------------|--------------------------------------| +| Creations | A new object has been created | +| Updates | An existing object has been modified | +| Deletions | An object has been deleted | +| Job starts | A job for an object starts | +| Job ends | A job for an object terminates | + +### Conditions + +A set of [prescribed conditions](../../reference/conditions.md) against which the triggering object will be evaluated. If the conditions are defined but not met by the object, no action will be taken. An event rule that does not define any conditions will _always_ trigger. diff --git a/docs/plugins/development/models.md b/docs/plugins/development/models.md index 46af135e11d..902ee9c82df 100644 --- a/docs/plugins/development/models.md +++ b/docs/plugins/development/models.md @@ -123,14 +123,17 @@ For more information about database migrations, see the [Django documentation](h ::: netbox.models.features.CustomValidationMixin +::: netbox.models.features.EventRulesMixin + +!!! note + `EventRulesMixin` was renamed from `WebhooksMixin` in NetBox v3.7. + ::: netbox.models.features.ExportTemplatesMixin ::: netbox.models.features.JournalingMixin ::: netbox.models.features.TagsMixin -::: netbox.models.features.WebhooksMixin - ## Choice Sets For model fields which support the selection of one or more values from a predefined list of choices, NetBox provides the `ChoiceSet` utility class. This can be used in place of a regular choices tuple to provide enhanced functionality, namely dynamic configuration and colorization. (See [Django's documentation](https://docs.djangoproject.com/en/stable/ref/models/fields/#choices) on the `choices` parameter for supported model fields.) diff --git a/mkdocs.yml b/mkdocs.yml index f927bf38665..8cbfd397be2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -87,6 +87,7 @@ nav: - Auth & Permissions: 'features/authentication-permissions.md' - API & Integration: 'features/api-integration.md' - Customization: 'features/customization.md' + - Event Rules: 'features/event-rules.md' - Installation & Upgrade: - Installing NetBox: 'installation/index.md' - 1. PostgreSQL: 'installation/1-postgresql.md' @@ -215,6 +216,7 @@ nav: - CustomField: 'models/extras/customfield.md' - CustomFieldChoiceSet: 'models/extras/customfieldchoiceset.md' - CustomLink: 'models/extras/customlink.md' + - EventRule: 'models/extras/eventrule.md' - ExportTemplate: 'models/extras/exporttemplate.md' - ImageAttachment: 'models/extras/imageattachment.md' - JournalEntry: 'models/extras/journalentry.md' diff --git a/netbox/core/models/contenttypes.py b/netbox/core/models/contenttypes.py index 0731871ec54..c98184c3d37 100644 --- a/netbox/core/models/contenttypes.py +++ b/netbox/core/models/contenttypes.py @@ -26,7 +26,7 @@ def with_feature(self, feature): Return the ContentTypes only for models which are registered as supporting the specified feature. For example, we can find all ContentTypes for models which support webhooks with - ContentType.objects.with_feature('webhooks') + ContentType.objects.with_feature('event_rules') """ if feature not in registry['model_features']: raise KeyError( diff --git a/netbox/core/models/jobs.py b/netbox/core/models/jobs.py index ce7ac6ec74e..af8191df500 100644 --- a/netbox/core/models/jobs.py +++ b/netbox/core/models/jobs.py @@ -16,7 +16,7 @@ from netbox.config import get_config from netbox.constants import RQ_QUEUE_DEFAULT from utilities.querysets import RestrictedQuerySet -from utilities.rqworker import get_queue_for_model, get_rq_retry +from utilities.rqworker import get_queue_for_model __all__ = ( 'Job', @@ -168,8 +168,8 @@ def start(self): self.status = JobStatusChoices.STATUS_RUNNING self.save() - # Handle webhooks - self.trigger_webhooks(event=EVENT_JOB_START) + # Handle events + self.process_event(event=EVENT_JOB_START) def terminate(self, status=JobStatusChoices.STATUS_COMPLETED, error=None): """ @@ -186,8 +186,8 @@ def terminate(self, status=JobStatusChoices.STATUS_COMPLETED, error=None): self.completed = timezone.now() self.save() - # Handle webhooks - self.trigger_webhooks(event=EVENT_JOB_END) + # Handle events + self.process_event(event=EVENT_JOB_END) @classmethod def enqueue(cls, func, instance, name='', user=None, schedule_at=None, interval=None, **kwargs): @@ -224,27 +224,18 @@ def enqueue(cls, func, instance, name='', user=None, schedule_at=None, interval= return job - def trigger_webhooks(self, event): - from extras.models import Webhook - - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = django_rq.get_queue(rq_queue_name, is_async=False) + def process_event(self, event): + """ + Process any EventRules relevant to the passed job event (i.e. start or stop). + """ + from extras.models import EventRule + from extras.events import process_event_rules - # Fetch any webhooks matching this object type and action - webhooks = Webhook.objects.filter( + # Fetch any event rules matching this object type and action + event_rules = EventRule.objects.filter( **{f'type_{event}': True}, content_types=self.object_type, enabled=True ) - for webhook in webhooks: - rq_queue.enqueue( - "extras.webhooks_worker.process_webhook", - webhook=webhook, - model_name=self.object_type.model, - event=event, - data=self.data, - timestamp=timezone.now().isoformat(), - username=self.user.username, - retry=get_rq_retry() - ) + process_event_rules(event_rules, self.object_type.model, event, self.data, self.user.username) diff --git a/netbox/extras/api/nested_serializers.py b/netbox/extras/api/nested_serializers.py index a97c630d25d..4bada494f8b 100644 --- a/netbox/extras/api/nested_serializers.py +++ b/netbox/extras/api/nested_serializers.py @@ -10,15 +10,25 @@ 'NestedCustomFieldChoiceSetSerializer', 'NestedCustomFieldSerializer', 'NestedCustomLinkSerializer', + 'NestedEventRuleSerializer', 'NestedExportTemplateSerializer', 'NestedImageAttachmentSerializer', 'NestedJournalEntrySerializer', 'NestedSavedFilterSerializer', + 'NestedScriptSerializer', 'NestedTagSerializer', # Defined in netbox.api.serializers 'NestedWebhookSerializer', ] +class NestedEventRuleSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:eventrule-detail') + + class Meta: + model = models.EventRule + fields = ['id', 'url', 'display', 'name'] + + class NestedWebhookSerializer(WritableNestedSerializer): url = serializers.HyperlinkedIdentityField(view_name='extras-api:webhook-detail') @@ -105,3 +115,20 @@ class NestedJournalEntrySerializer(WritableNestedSerializer): class Meta: model = models.JournalEntry fields = ['id', 'url', 'display', 'created'] + + +class NestedScriptSerializer(WritableNestedSerializer): + url = serializers.HyperlinkedIdentityField( + view_name='extras-api:script-detail', + lookup_field='full_name', + lookup_url_kwarg='pk' + ) + name = serializers.CharField(read_only=True) + display = serializers.SerializerMethodField(read_only=True) + + class Meta: + model = models.Script + fields = ['id', 'url', 'display', 'name'] + + def get_display(self, obj): + return f'{obj.name} ({obj.module})' diff --git a/netbox/extras/api/serializers.py b/netbox/extras/api/serializers.py index 4e1b47503d0..82b3e1933a6 100644 --- a/netbox/extras/api/serializers.py +++ b/netbox/extras/api/serializers.py @@ -1,17 +1,17 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from core.api.serializers import JobSerializer from core.api.nested_serializers import NestedDataSourceSerializer, NestedDataFileSerializer, NestedJobSerializer +from core.api.serializers import JobSerializer from core.models import ContentType from dcim.api.nested_serializers import ( NestedDeviceRoleSerializer, NestedDeviceTypeSerializer, NestedLocationSerializer, NestedPlatformSerializer, NestedRegionSerializer, NestedSiteSerializer, NestedSiteGroupSerializer, ) from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup -from drf_spectacular.utils import extend_schema_field -from drf_spectacular.types import OpenApiTypes from extras.choices import * from extras.models import * from netbox.api.exceptions import SerializerNotFound @@ -38,6 +38,7 @@ 'CustomFieldSerializer', 'CustomLinkSerializer', 'DashboardSerializer', + 'EventRuleSerializer', 'ExportTemplateSerializer', 'ImageAttachmentSerializer', 'JournalEntrySerializer', @@ -57,23 +58,58 @@ # -# Webhooks +# Event Rules # -class WebhookSerializer(NetBoxModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='extras-api:webhook-detail') +class EventRuleSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:eventrule-detail') content_types = ContentTypeField( - queryset=ContentType.objects.with_feature('webhooks'), + queryset=ContentType.objects.with_feature('event_rules'), many=True ) + action_type = ChoiceField(choices=EventRuleActionChoices) + action_object_type = ContentTypeField( + queryset=ContentType.objects.with_feature('event_rules'), + ) + action_object = serializers.SerializerMethodField(read_only=True) class Meta: - model = Webhook + model = EventRule fields = [ 'id', 'url', 'display', 'content_types', 'name', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'payload_url', 'enabled', 'http_method', 'http_content_type', - 'additional_headers', 'body_template', 'secret', 'conditions', 'ssl_verification', 'ca_file_path', - 'custom_fields', 'tags', 'created', 'last_updated', + 'type_job_start', 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', + 'action_object_id', 'action_object', 'description', 'custom_fields', 'tags', 'created', 'last_updated', + ] + + @extend_schema_field(OpenApiTypes.OBJECT) + def get_action_object(self, instance): + context = {'request': self.context['request']} + # We need to manually instantiate the serializer for scripts + if instance.action_type == EventRuleActionChoices.SCRIPT: + module_id, script_name = instance.action_parameters['script_choice'].split(":", maxsplit=1) + script = instance.action_object.scripts[script_name]() + return NestedScriptSerializer(script, context=context).data + else: + serializer = get_serializer_for_model( + model=instance.action_object_type.model_class(), + prefix=NESTED_SERIALIZER_PREFIX + ) + return serializer(instance.action_object, context=context).data + + +# +# Webhooks +# + +class WebhookSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField(view_name='extras-api:webhook-detail') + + class Meta: + model = Webhook + fields = [ + 'id', 'url', 'display', 'name', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', + 'body_template', 'secret', 'ssl_verification', 'ca_file_path', 'custom_fields', 'tags', 'created', + 'last_updated', ] diff --git a/netbox/extras/api/urls.py b/netbox/extras/api/urls.py index 5f2b324e6d4..1616b85549e 100644 --- a/netbox/extras/api/urls.py +++ b/netbox/extras/api/urls.py @@ -7,6 +7,7 @@ router = NetBoxRouter() router.APIRootView = views.ExtrasRootView +router.register('event-rules', views.EventRuleViewSet) router.register('webhooks', views.WebhookViewSet) router.register('custom-fields', views.CustomFieldViewSet) router.register('custom-field-choice-sets', views.CustomFieldChoiceSetViewSet) diff --git a/netbox/extras/api/views.py b/netbox/extras/api/views.py index 830982e7455..e0fca86177f 100644 --- a/netbox/extras/api/views.py +++ b/netbox/extras/api/views.py @@ -37,6 +37,17 @@ def get_view_name(self): return 'Extras' +# +# EventRules +# + +class EventRuleViewSet(NetBoxModelViewSet): + metadata_class = ContentTypeMetadata + queryset = EventRule.objects.all() + serializer_class = serializers.EventRuleSerializer + filterset_class = filtersets.EventRuleFilterSet + + # # Webhooks # diff --git a/netbox/extras/choices.py b/netbox/extras/choices.py index fdb951b7ddf..14179fb397c 100644 --- a/netbox/extras/choices.py +++ b/netbox/extras/choices.py @@ -291,3 +291,18 @@ class DashboardWidgetColorChoices(ChoiceSet): (BLACK, _('Black')), (WHITE, _('White')), ) + + +# +# Event Rules +# + +class EventRuleActionChoices(ChoiceSet): + + WEBHOOK = 'webhook' + SCRIPT = 'script' + + CHOICES = ( + (WEBHOOK, _('Webhook')), + (SCRIPT, _('Script')), + ) diff --git a/netbox/extras/context_managers.py b/netbox/extras/context_managers.py index 32323999efe..8de47465e96 100644 --- a/netbox/extras/context_managers.py +++ b/netbox/extras/context_managers.py @@ -1,25 +1,25 @@ from contextlib import contextmanager -from netbox.context import current_request, webhooks_queue -from .webhooks import flush_webhooks +from netbox.context import current_request, events_queue +from .events import flush_events @contextmanager -def change_logging(request): +def event_tracking(request): """ - Enable change logging by connecting the appropriate signals to their receivers before code is run, and - disconnecting them afterward. + Queue interesting events in memory while processing a request, then flush that queue for processing by the + events pipline before returning the response. :param request: WSGIRequest object with a unique `id` set """ current_request.set(request) - webhooks_queue.set([]) + events_queue.set([]) yield # Flush queued webhooks to RQ - flush_webhooks(webhooks_queue.get()) + flush_events(events_queue.get()) # Clear context vars current_request.set(None) - webhooks_queue.set([]) + events_queue.set([]) diff --git a/netbox/extras/events.py b/netbox/extras/events.py new file mode 100644 index 00000000000..05352b7d197 --- /dev/null +++ b/netbox/extras/events.py @@ -0,0 +1,178 @@ +import logging + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist +from django.utils import timezone +from django.utils.module_loading import import_string +from django_rq import get_queue + +from core.models import Job +from netbox.config import get_config +from netbox.constants import RQ_QUEUE_DEFAULT +from netbox.registry import registry +from utilities.api import get_serializer_for_model +from utilities.rqworker import get_rq_retry +from utilities.utils import serialize_object +from .choices import * +from .models import EventRule, ScriptModule + +logger = logging.getLogger('netbox.events_processor') + + +def serialize_for_event(instance): + """ + Return a serialized representation of the given instance suitable for use in a queued event. + """ + serializer_class = get_serializer_for_model(instance.__class__) + serializer_context = { + 'request': None, + } + serializer = serializer_class(instance, context=serializer_context) + + return serializer.data + + +def get_snapshots(instance, action): + snapshots = { + 'prechange': getattr(instance, '_prechange_snapshot', None), + 'postchange': None, + } + if action != ObjectChangeActionChoices.ACTION_DELETE: + # Use model's serialize_object() method if defined; fall back to serialize_object() utility function + if hasattr(instance, 'serialize_object'): + snapshots['postchange'] = instance.serialize_object() + else: + snapshots['postchange'] = serialize_object(instance) + + return snapshots + + +def enqueue_object(queue, instance, user, request_id, action): + """ + Enqueue a serialized representation of a created/updated/deleted object for the processing of + events once the request has completed. + """ + # Determine whether this type of object supports event rules + app_label = instance._meta.app_label + model_name = instance._meta.model_name + if model_name not in registry['model_features']['event_rules'].get(app_label, []): + return + + queue.append({ + 'content_type': ContentType.objects.get_for_model(instance), + 'object_id': instance.pk, + 'event': action, + 'data': serialize_for_event(instance), + 'snapshots': get_snapshots(instance, action), + 'username': user.username, + 'request_id': request_id + }) + + +def process_event_rules(event_rules, model_name, event, data, username, snapshots=None, request_id=None): + try: + user = get_user_model().objects.get(username=username) + except ObjectDoesNotExist: + user = None + + for event_rule in event_rules: + + # Evaluate event rule conditions (if any) + if not event_rule.eval_conditions(data): + return + + # Webhooks + if event_rule.action_type == EventRuleActionChoices.WEBHOOK: + + # Select the appropriate RQ queue + queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) + rq_queue = get_queue(queue_name) + + # Compile the task parameters + params = { + "event_rule": event_rule, + "model_name": model_name, + "event": event, + "data": data, + "snapshots": snapshots, + "timestamp": timezone.now().isoformat(), + "username": username, + "retry": get_rq_retry() + } + if snapshots: + params["snapshots"] = snapshots + if request_id: + params["request_id"] = request_id + + # Enqueue the task + rq_queue.enqueue( + "extras.webhooks_worker.process_webhook", + **params + ) + + # Scripts + elif event_rule.action_type == EventRuleActionChoices.SCRIPT: + # Resolve the script from action parameters + script_module = event_rule.action_object + _, script_name = event_rule.action_parameters['script_choice'].split(":", maxsplit=1) + script = script_module.scripts[script_name]() + + # Enqueue a Job to record the script's execution + Job.enqueue( + "extras.scripts.run_script", + instance=script_module, + name=script.class_name, + user=user, + data=data + ) + + else: + raise ValueError(f"Unknown action type for an event rule: {event_rule.action_type}") + + +def process_event_queue(events): + """ + Flush a list of object representation to RQ for EventRule processing. + """ + events_cache = { + 'type_create': {}, + 'type_update': {}, + 'type_delete': {}, + } + + for data in events: + action_flag = { + ObjectChangeActionChoices.ACTION_CREATE: 'type_create', + ObjectChangeActionChoices.ACTION_UPDATE: 'type_update', + ObjectChangeActionChoices.ACTION_DELETE: 'type_delete', + }[data['event']] + content_type = data['content_type'] + + # Cache applicable Event Rules + if content_type not in events_cache[action_flag]: + events_cache[action_flag][content_type] = EventRule.objects.filter( + **{action_flag: True}, + content_types=content_type, + enabled=True + ) + event_rules = events_cache[action_flag][content_type] + + process_event_rules( + event_rules, content_type.model, data['event'], data['data'], data['username'], + snapshots=data['snapshots'], request_id=data['request_id'] + ) + + +def flush_events(queue): + """ + Flush a list of object representation to RQ for webhook processing. + """ + if queue: + for name in settings.EVENTS_PIPELINE: + try: + func = import_string(name) + func(queue) + except Exception as e: + logger.error(f"Cannot import events pipeline {name} error: {e}") diff --git a/netbox/extras/filtersets.py b/netbox/extras/filtersets.py index e0fc44ab112..e3eeda20d58 100644 --- a/netbox/extras/filtersets.py +++ b/netbox/extras/filtersets.py @@ -22,6 +22,7 @@ 'CustomFieldChoiceSetFilterSet', 'CustomFieldFilterSet', 'CustomLinkFilterSet', + 'EventRuleFilterSet', 'ExportTemplateFilterSet', 'ImageAttachmentFilterSet', 'JournalEntryFilterSet', @@ -38,19 +39,18 @@ class WebhookFilterSet(NetBoxModelFilterSet): method='search', label=_('Search'), ) - content_type_id = MultiValueNumberFilter( - field_name='content_types__id' - ) - content_types = ContentTypeFilter() http_method = django_filters.MultipleChoiceFilter( choices=WebhookHttpMethodChoices ) + payload_url = MultiValueCharFilter( + lookup_expr='icontains' + ) class Meta: model = Webhook fields = [ - 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'payload_url', - 'enabled', 'http_method', 'http_content_type', 'secret', 'ssl_verification', 'ca_file_path', + 'id', 'name', 'payload_url', 'http_method', 'http_content_type', 'secret', 'ssl_verification', + 'ca_file_path', ] def search(self, queryset, name, value): @@ -62,6 +62,38 @@ def search(self, queryset, name, value): ) +class EventRuleFilterSet(NetBoxModelFilterSet): + q = django_filters.CharFilter( + method='search', + label=_('Search'), + ) + content_type_id = MultiValueNumberFilter( + field_name='content_types__id' + ) + content_types = ContentTypeFilter() + action_type = django_filters.MultipleChoiceFilter( + choices=EventRuleActionChoices + ) + action_object_type = ContentTypeFilter() + action_object_id = MultiValueNumberFilter() + + class Meta: + model = EventRule + fields = [ + 'id', 'name', 'type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end', 'enabled', + 'action_type', 'description', + ] + + def search(self, queryset, name, value): + if not value.strip(): + return queryset + return queryset.filter( + Q(name__icontains=value) | + Q(description__icontains=value) | + Q(comments__icontains=value) + ) + + class CustomFieldFilterSet(BaseFilterSet): q = django_filters.CharFilter( method='search', diff --git a/netbox/extras/forms/bulk_edit.py b/netbox/extras/forms/bulk_edit.py index 5da2a5ddeca..dade76bade6 100644 --- a/netbox/extras/forms/bulk_edit.py +++ b/netbox/extras/forms/bulk_edit.py @@ -14,6 +14,7 @@ 'CustomFieldBulkEditForm', 'CustomFieldChoiceSetBulkEditForm', 'CustomLinkBulkEditForm', + 'EventRuleBulkEditForm', 'ExportTemplateBulkEditForm', 'JournalEntryBulkEditForm', 'SavedFilterBulkEditForm', @@ -177,6 +178,39 @@ class WebhookBulkEditForm(NetBoxModelBulkEditForm): queryset=Webhook.objects.all(), widget=forms.MultipleHiddenInput ) + http_method = forms.ChoiceField( + choices=add_blank_choice(WebhookHttpMethodChoices), + required=False, + label=_('HTTP method') + ) + payload_url = forms.CharField( + required=False, + label=_('Payload URL') + ) + ssl_verification = forms.NullBooleanField( + required=False, + widget=BulkEditNullBooleanSelect(), + label=_('SSL verification') + ) + secret = forms.CharField( + label=_('Secret'), + required=False + ) + ca_file_path = forms.CharField( + required=False, + label=_('CA file path') + ) + + nullable_fields = ('secret', 'ca_file_path') + + +class EventRuleBulkEditForm(NetBoxModelBulkEditForm): + model = EventRule + + pk = forms.ModelMultipleChoiceField( + queryset=EventRule.objects.all(), + widget=forms.MultipleHiddenInput + ) enabled = forms.NullBooleanField( label=_('Enabled'), required=False, @@ -207,30 +241,8 @@ class WebhookBulkEditForm(NetBoxModelBulkEditForm): required=False, widget=BulkEditNullBooleanSelect() ) - http_method = forms.ChoiceField( - choices=add_blank_choice(WebhookHttpMethodChoices), - required=False, - label=_('HTTP method') - ) - payload_url = forms.CharField( - required=False, - label=_('Payload URL') - ) - ssl_verification = forms.NullBooleanField( - required=False, - widget=BulkEditNullBooleanSelect(), - label=_('SSL verification') - ) - secret = forms.CharField( - label=_('Secret'), - required=False - ) - ca_file_path = forms.CharField( - required=False, - label=_('CA file path') - ) - nullable_fields = ('secret', 'conditions', 'ca_file_path') + nullable_fields = ('conditions',) class TagBulkEditForm(BulkEditForm): diff --git a/netbox/extras/forms/bulk_import.py b/netbox/extras/forms/bulk_import.py index 181b1f8d3eb..82930e8ad6e 100644 --- a/netbox/extras/forms/bulk_import.py +++ b/netbox/extras/forms/bulk_import.py @@ -1,5 +1,6 @@ from django import forms from django.contrib.postgres.forms import SimpleArrayField +from django.core.exceptions import ObjectDoesNotExist from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -17,6 +18,7 @@ 'CustomFieldChoiceSetImportForm', 'CustomFieldImportForm', 'CustomLinkImportForm', + 'EventRuleImportForm', 'ExportTemplateImportForm', 'JournalEntryImportForm', 'SavedFilterImportForm', @@ -143,20 +145,61 @@ class Meta: class WebhookImportForm(NetBoxModelImportForm): + + class Meta: + model = Webhook + fields = ( + 'name', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', + 'secret', 'ssl_verification', 'ca_file_path', 'tags' + ) + + +class EventRuleImportForm(NetBoxModelImportForm): content_types = CSVMultipleContentTypeField( label=_('Content types'), - queryset=ContentType.objects.with_feature('webhooks'), + queryset=ContentType.objects.with_feature('event_rules'), help_text=_("One or more assigned object types") ) + action_object = forms.CharField( + label=_('Action object'), + required=True, + help_text=_('Webhook name or script as dotted path module.Class') + ) class Meta: - model = Webhook + model = EventRule fields = ( - 'name', 'enabled', 'content_types', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', - 'secret', 'ssl_verification', 'ca_file_path', 'tags' + 'name', 'description', 'enabled', 'conditions', 'content_types', 'type_create', 'type_update', + 'type_delete', 'type_job_start', 'type_job_end', 'action_type', 'action_object', 'comments', 'tags' ) + def clean(self): + super().clean() + + action_object = self.cleaned_data.get('action_object') + action_type = self.cleaned_data.get('action_type') + if action_object and action_type: + if action_type == EventRuleActionChoices.WEBHOOK: + try: + webhook = Webhook.objects.get(name=action_object) + except Webhook.ObjectDoesNotExist: + raise forms.ValidationError(f"Webhook {action_object} not found") + self.instance.action_object = webhook + elif action_type == EventRuleActionChoices.SCRIPT: + from extras.scripts import get_module_and_script + module_name, script_name = action_object.split('.', 1) + try: + module, script = get_module_and_script(module_name, script_name) + except ObjectDoesNotExist: + raise forms.ValidationError(f"Script {action_object} not found") + self.instance.action_object = module + self.instance.action_object_type = ContentType.objects.get_for_model(module, for_concrete_model=False) + self.instance.action_parameters = { + 'script_choice': f"{str(module.pk)}:{script_name}", + 'script_name': script.name, + 'script_full_name': script.full_name, + } + class TagImportForm(CSVModelForm): slug = SlugField() diff --git a/netbox/extras/forms/filtersets.py b/netbox/extras/forms/filtersets.py index b68845c2feb..c91e3b8c6c7 100644 --- a/netbox/extras/forms/filtersets.py +++ b/netbox/extras/forms/filtersets.py @@ -22,6 +22,7 @@ 'CustomFieldChoiceSetFilterForm', 'CustomFieldFilterForm', 'CustomLinkFilterForm', + 'EventRuleFilterForm', 'ExportTemplateFilterForm', 'ImageAttachmentFilterForm', 'JournalEntryFilterForm', @@ -223,22 +224,44 @@ class SavedFilterFilterForm(SavedFiltersMixin, FilterForm): class WebhookFilterForm(NetBoxModelFilterSetForm): model = Webhook + fieldsets = ( + (None, ('q', 'filter_id', 'tag')), + (_('Attributes'), ('payload_url', 'http_method', 'http_content_type')), + ) + http_content_type = forms.CharField( + label=_('HTTP content type'), + required=False + ) + payload_url = forms.CharField( + label=_('Payload URL'), + required=False + ) + http_method = forms.MultipleChoiceField( + choices=WebhookHttpMethodChoices, + required=False, + label=_('HTTP method') + ) + tag = TagFilterField(model) + + +class EventRuleFilterForm(NetBoxModelFilterSetForm): + model = EventRule tag = TagFilterField(model) fieldsets = ( (None, ('q', 'filter_id', 'tag')), - (_('Attributes'), ('content_type_id', 'http_method', 'enabled')), + (_('Attributes'), ('content_type_id', 'action_type', 'enabled')), (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), ) content_type_id = ContentTypeMultipleChoiceField( - queryset=ContentType.objects.with_feature('webhooks'), + queryset=ContentType.objects.with_feature('event_rules'), required=False, label=_('Object type') ) - http_method = forms.MultipleChoiceField( - choices=WebhookHttpMethodChoices, + action_type = forms.ChoiceField( + choices=add_blank_choice(EventRuleActionChoices), required=False, - label=_('HTTP method') + label=_('Action type') ) enabled = forms.NullBooleanField( label=_('Enabled'), diff --git a/netbox/extras/forms/model_forms.py b/netbox/extras/forms/model_forms.py index 9553a839a79..0c717246f2b 100644 --- a/netbox/extras/forms/model_forms.py +++ b/netbox/extras/forms/model_forms.py @@ -1,6 +1,7 @@ import json from django import forms +from django.contrib.contenttypes.models import ContentType from django.utils.safestring import mark_safe from django.utils.translation import gettext_lazy as _ @@ -11,12 +12,12 @@ from extras.models import * from netbox.forms import NetBoxModelForm from tenancy.models import Tenant, TenantGroup -from utilities.forms import BootstrapMixin, add_blank_choice +from utilities.forms import BootstrapMixin, add_blank_choice, get_field_value from utilities.forms.fields import ( CommentField, ContentTypeChoiceField, ContentTypeMultipleChoiceField, DynamicModelChoiceField, DynamicModelMultipleChoiceField, JSONField, SlugField, ) -from utilities.forms.widgets import ChoicesWidget +from utilities.forms.widgets import ChoicesWidget, HTMXSelect from virtualization.models import Cluster, ClusterGroup, ClusterType __all__ = ( @@ -26,6 +27,7 @@ 'CustomFieldChoiceSetForm', 'CustomFieldForm', 'CustomLinkForm', + 'EventRuleForm', 'ExportTemplateForm', 'ImageAttachmentForm', 'JournalEntryForm', @@ -211,24 +213,59 @@ class Meta: class WebhookForm(NetBoxModelForm): - content_types = ContentTypeMultipleChoiceField( - label=_('Content types'), - queryset=ContentType.objects.with_feature('webhooks') - ) fieldsets = ( - (_('Webhook'), ('name', 'content_types', 'enabled', 'tags')), - (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + (_('Webhook'), ('name', 'tags',)), (_('HTTP Request'), ( 'payload_url', 'http_method', 'http_content_type', 'additional_headers', 'body_template', 'secret', )), - (_('Conditions'), ('conditions',)), (_('SSL'), ('ssl_verification', 'ca_file_path')), ) class Meta: model = Webhook fields = '__all__' + widgets = { + 'additional_headers': forms.Textarea(attrs={'class': 'font-monospace'}), + 'body_template': forms.Textarea(attrs={'class': 'font-monospace'}), + } + + +class EventRuleForm(NetBoxModelForm): + content_types = ContentTypeMultipleChoiceField( + label=_('Content types'), + queryset=ContentType.objects.with_feature('event_rules'), + ) + action_choice = forms.ChoiceField( + label=_('Action choice'), + choices=[] + ) + conditions = JSONField( + required=False, + help_text=_('Enter conditions in JSON format.') + ) + action_data = JSONField( + required=False, + help_text=_('Enter parameters to pass to the action in JSON format.') + ) + + fieldsets = ( + (_('Event Rule'), ('name', 'description', 'content_types', 'enabled', 'tags')), + (_('Events'), ('type_create', 'type_update', 'type_delete', 'type_job_start', 'type_job_end')), + (_('Conditions'), ('conditions',)), + (_('Action'), ( + 'action_type', 'action_choice', 'action_parameters', 'action_object_type', 'action_object_id', + 'action_data', + )), + ) + + class Meta: + model = EventRule + fields = ( + 'content_types', 'name', 'description', 'type_create', 'type_update', 'type_delete', 'type_job_start', + 'type_job_end', 'enabled', 'conditions', 'action_type', 'action_object_type', 'action_object_id', + 'action_parameters', 'action_data', 'comments', 'tags' + ) labels = { 'type_create': _('Creations'), 'type_update': _('Updates'), @@ -237,11 +274,76 @@ class Meta: 'type_job_end': _('Job terminations'), } widgets = { - 'additional_headers': forms.Textarea(attrs={'class': 'font-monospace'}), - 'body_template': forms.Textarea(attrs={'class': 'font-monospace'}), 'conditions': forms.Textarea(attrs={'class': 'font-monospace'}), + 'action_type': HTMXSelect(), + 'action_object_type': forms.HiddenInput, + 'action_object_id': forms.HiddenInput, + 'action_parameters': forms.HiddenInput, } + def init_script_choice(self): + choices = [] + for module in ScriptModule.objects.all(): + scripts = [] + for script_name in module.scripts.keys(): + name = f"{str(module.pk)}:{script_name}" + scripts.append((name, script_name)) + + if scripts: + choices.append((str(module), scripts)) + + self.fields['action_choice'].choices = choices + parameters = get_field_value(self, 'action_parameters') + initial = None + if parameters and 'script_choice' in parameters: + initial = parameters['script_choice'] + self.fields['action_choice'].initial = initial + + def init_webhook_choice(self): + initial = None + if self.fields['action_object_type'] and get_field_value(self, 'action_object_id'): + initial = Webhook.objects.get(pk=get_field_value(self, 'action_object_id')) + self.fields['action_choice'] = DynamicModelChoiceField( + label=_('Webhook'), + queryset=Webhook.objects.all(), + required=True, + initial=initial + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields['action_object_type'].required = False + self.fields['action_object_id'].required = False + + # Determine the action type + action_type = get_field_value(self, 'action_type') + + if action_type == EventRuleActionChoices.WEBHOOK: + self.init_webhook_choice() + elif action_type == EventRuleActionChoices.SCRIPT: + self.init_script_choice() + + def clean(self): + super().clean() + + action_choice = self.cleaned_data.get('action_choice') + if self.cleaned_data.get('action_type') == EventRuleActionChoices.WEBHOOK: + self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(action_choice) + self.cleaned_data['action_object_id'] = action_choice.id + elif self.cleaned_data.get('action_type') == EventRuleActionChoices.SCRIPT: + module_id, script_name = action_choice.split(":", maxsplit=1) + script_module = ScriptModule.objects.get(pk=module_id) + self.cleaned_data['action_object_type'] = ContentType.objects.get_for_model(script_module, for_concrete_model=False) + self.cleaned_data['action_object_id'] = script_module.id + script = script_module.scripts[script_name]() + self.cleaned_data['action_parameters'] = { + 'script_choice': action_choice, + 'script_name': script.name, + 'script_full_name': script.full_name, + } + + return self.cleaned_data + class TagForm(BootstrapMixin, forms.ModelForm): slug = SlugField() diff --git a/netbox/extras/graphql/schema.py b/netbox/extras/graphql/schema.py index e13cc0e9f46..09e399e370e 100644 --- a/netbox/extras/graphql/schema.py +++ b/netbox/extras/graphql/schema.py @@ -72,3 +72,9 @@ def resolve_tag_list(root, info, **kwargs): def resolve_webhook_list(root, info, **kwargs): return gql_query_optimizer(models.Webhook.objects.all(), info) + + event_rule = ObjectField(EventRuleType) + event_rule_list = ObjectListField(EventRuleType) + + def resolve_eventrule_list(root, info, **kwargs): + return gql_query_optimizer(models.EventRule.objects.all(), info) diff --git a/netbox/extras/graphql/types.py b/netbox/extras/graphql/types.py index 068da02f2bc..4981ddd7206 100644 --- a/netbox/extras/graphql/types.py +++ b/netbox/extras/graphql/types.py @@ -8,6 +8,7 @@ 'CustomFieldChoiceSetType', 'CustomFieldType', 'CustomLinkType', + 'EventRuleType', 'ExportTemplateType', 'ImageAttachmentType', 'JournalEntryType', @@ -110,5 +111,12 @@ class WebhookType(OrganizationalObjectType): class Meta: model = models.Webhook - exclude = ('content_types', ) filterset_class = filtersets.WebhookFilterSet + + +class EventRuleType(OrganizationalObjectType): + + class Meta: + model = models.EventRule + exclude = ('content_types', ) + filterset_class = filtersets.EventRuleFilterSet diff --git a/netbox/extras/management/commands/runscript.py b/netbox/extras/management/commands/runscript.py index 3cf70281cb6..97ee39f5011 100644 --- a/netbox/extras/management/commands/runscript.py +++ b/netbox/extras/management/commands/runscript.py @@ -11,9 +11,9 @@ from core.choices import JobStatusChoices from core.models import Job from extras.api.serializers import ScriptOutputSerializer -from extras.context_managers import change_logging +from extras.context_managers import event_tracking from extras.scripts import get_module_and_script -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.exceptions import AbortTransaction from utilities.utils import NetBoxFakeRequest @@ -37,7 +37,7 @@ def handle(self, *args, **options): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the change_logging context manager (which is bypassed if commit == False). + the event_tracking context manager (which is bypassed if commit == False). """ try: try: @@ -47,7 +47,7 @@ def _run_script(): raise AbortTransaction() except AbortTransaction: script.log_info("Database changes have been reverted automatically.") - clear_webhooks.send(request) + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate() except Exception as e: @@ -57,7 +57,7 @@ def _run_script(): ) script.log_info("Database changes have been reverted due to error.") logger.error(f"Exception raised during script execution: {e}") - clear_webhooks.send(request) + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate(status=JobStatusChoices.STATUS_ERRORED, error=str(e)) @@ -136,9 +136,9 @@ def _run_script(): logger.info(f"Running script (commit={commit})") script.request = request - # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process + # Execute the script. If commit is True, wrap it with the event_tracking context manager to ensure we process # change logging, webhooks, etc. - with change_logging(request): + with event_tracking(request): _run_script() else: logger.error('Data is not valid:') diff --git a/netbox/extras/migrations/0101_eventrule.py b/netbox/extras/migrations/0101_eventrule.py new file mode 100644 index 00000000000..64e03dda01c --- /dev/null +++ b/netbox/extras/migrations/0101_eventrule.py @@ -0,0 +1,127 @@ +import django.db.models.deletion +import taggit.managers +from django.contrib.contenttypes.models import ContentType +from django.db import migrations, models + +import utilities.json +from extras.choices import * + + +def move_webhooks(apps, schema_editor): + Webhook = apps.get_model("extras", "Webhook") + EventRule = apps.get_model("extras", "EventRule") + + for webhook in Webhook.objects.all(): + event = EventRule() + + event.name = webhook.name + event.type_create = webhook.type_create + event.type_update = webhook.type_update + event.type_delete = webhook.type_delete + event.type_job_start = webhook.type_job_start + event.type_job_end = webhook.type_job_end + event.enabled = webhook.enabled + event.conditions = webhook.conditions + + event.action_type = EventRuleActionChoices.WEBHOOK + event.action_object_type_id = ContentType.objects.get_for_model(webhook).id + event.action_object_id = webhook.id + event.save() + event.content_types.add(*webhook.content_types.all()) + + +class Migration(migrations.Migration): + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('extras', '0100_customfield_ui_attrs'), + ] + + operations = [ + migrations.CreateModel( + name='EventRule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)), + ('created', models.DateTimeField(auto_now_add=True, null=True)), + ('last_updated', models.DateTimeField(auto_now=True, null=True)), + ( + 'custom_field_data', + models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder), + ), + ('name', models.CharField(max_length=150, unique=True)), + ('description', models.CharField(blank=True, max_length=200)), + ('type_create', models.BooleanField(default=False)), + ('type_update', models.BooleanField(default=False)), + ('type_delete', models.BooleanField(default=False)), + ('type_job_start', models.BooleanField(default=False)), + ('type_job_end', models.BooleanField(default=False)), + ('enabled', models.BooleanField(default=True)), + ('conditions', models.JSONField(blank=True, null=True)), + ('action_type', models.CharField(default='webhook', max_length=30)), + ('action_object_id', models.PositiveBigIntegerField(blank=True, null=True)), + ('action_parameters', models.JSONField(blank=True, null=True)), + ('action_data', models.JSONField(blank=True, null=True)), + ('comments', models.TextField(blank=True)), + ], + options={ + 'verbose_name': 'eventrule', + 'verbose_name_plural': 'eventrules', + 'ordering': ('name',), + }, + ), + migrations.RunPython(move_webhooks), + migrations.RemoveConstraint( + model_name='webhook', + name='extras_webhook_unique_payload_url_types', + ), + migrations.RemoveField( + model_name='webhook', + name='conditions', + ), + migrations.RemoveField( + model_name='webhook', + name='content_types', + ), + migrations.RemoveField( + model_name='webhook', + name='enabled', + ), + migrations.RemoveField( + model_name='webhook', + name='type_create', + ), + migrations.RemoveField( + model_name='webhook', + name='type_delete', + ), + migrations.RemoveField( + model_name='webhook', + name='type_job_end', + ), + migrations.RemoveField( + model_name='webhook', + name='type_job_start', + ), + migrations.RemoveField( + model_name='webhook', + name='type_update', + ), + migrations.AddField( + model_name='eventrule', + name='action_object_type', + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name='eventrule_actions', + to='contenttypes.contenttype', + ), + ), + migrations.AddField( + model_name='eventrule', + name='content_types', + field=models.ManyToManyField(related_name='eventrules', to='contenttypes.contenttype'), + ), + migrations.AddField( + model_name='eventrule', + name='tags', + field=taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag'), + ), + ] diff --git a/netbox/extras/migrations/0101_move_configrevision.py b/netbox/extras/migrations/0102_move_configrevision.py similarity index 96% rename from netbox/extras/migrations/0101_move_configrevision.py rename to netbox/extras/migrations/0102_move_configrevision.py index 730e7a09648..36eef12059e 100644 --- a/netbox/extras/migrations/0101_move_configrevision.py +++ b/netbox/extras/migrations/0102_move_configrevision.py @@ -15,7 +15,7 @@ def update_content_type(apps, schema_editor): class Migration(migrations.Migration): dependencies = [ - ('extras', '0100_customfield_ui_attrs'), + ('extras', '0101_eventrule'), ] operations = [ diff --git a/netbox/extras/models/models.py b/netbox/extras/models/models.py index d0a2e4b61da..e5f71dba3c4 100644 --- a/netbox/extras/models/models.py +++ b/netbox/extras/models/models.py @@ -2,7 +2,7 @@ import urllib.parse from django.conf import settings -from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation from django.core.validators import ValidationError from django.db import models from django.http import HttpResponse @@ -28,6 +28,7 @@ __all__ = ( 'Bookmark', 'CustomLink', + 'EventRule', 'ExportTemplate', 'ImageAttachment', 'JournalEntry', @@ -36,23 +37,28 @@ ) -class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): +class EventRule(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): """ - A Webhook defines a request that will be sent to a remote application when an object is created, updated, and/or - delete in NetBox. The request will contain a representation of the object, which the remote application can act on. - Each Webhook can be limited to firing only on certain actions or certain object types. + An EventRule defines an action to be taken automatically in response to a specific set of events, such as when a + specific type of object is created, modified, or deleted. The action to be taken might entail transmitting a + webhook or executing a custom script. """ content_types = models.ManyToManyField( to='contenttypes.ContentType', - related_name='webhooks', + related_name='eventrules', verbose_name=_('object types'), - help_text=_("The object(s) to which this Webhook applies.") + help_text=_("The object(s) to which this rule applies.") ) name = models.CharField( verbose_name=_('name'), max_length=150, unique=True ) + description = models.CharField( + verbose_name=_('description'), + max_length=200, + blank=True + ) type_create = models.BooleanField( verbose_name=_('on create'), default=False, @@ -78,6 +84,104 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo default=False, help_text=_("Triggers when a job for a matching object terminates.") ) + enabled = models.BooleanField( + verbose_name=_('enabled'), + default=True + ) + conditions = models.JSONField( + verbose_name=_('conditions'), + blank=True, + null=True, + help_text=_("A set of conditions which determine whether the event will be generated.") + ) + + # Action to take + action_type = models.CharField( + max_length=30, + choices=EventRuleActionChoices, + default=EventRuleActionChoices.WEBHOOK, + verbose_name=_('action type') + ) + action_object_type = models.ForeignKey( + to='contenttypes.ContentType', + related_name='eventrule_actions', + on_delete=models.CASCADE + ) + action_object_id = models.PositiveBigIntegerField( + blank=True, + null=True + ) + action_object = GenericForeignKey( + ct_field='action_object_type', + fk_field='action_object_id' + ) + # internal (not show in UI) - used by scripts to store function name + action_parameters = models.JSONField( + blank=True, + null=True, + ) + action_data = models.JSONField( + verbose_name=_('parameters'), + blank=True, + null=True, + help_text=_("Parameters to pass to the action.") + ) + comments = models.TextField( + verbose_name=_('comments'), + blank=True + ) + + class Meta: + ordering = ('name',) + verbose_name = _('event rule') + verbose_name_plural = _('event rules') + + def __str__(self): + return self.name + + def get_absolute_url(self): + return reverse('extras:eventrule', args=[self.pk]) + + def clean(self): + super().clean() + + # At least one action type must be selected + if not any([ + self.type_create, self.type_update, self.type_delete, self.type_job_start, self.type_job_end + ]): + raise ValidationError( + _("At least one event type must be selected: create, update, delete, job start, and/or job end.") + ) + + # Validate that any conditions are in the correct format + if self.conditions: + try: + ConditionSet(self.conditions) + except ValueError as e: + raise ValidationError({'conditions': e}) + + def eval_conditions(self, data): + """ + Test whether the given data meets the conditions of the event rule (if any). Return True + if met or no conditions are specified. + """ + if not self.conditions: + return True + + return ConditionSet(self.conditions).eval(data) + + +class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedModel): + """ + A Webhook defines a request that will be sent to a remote application when an object is created, updated, and/or + delete in NetBox. The request will contain a representation of the object, which the remote application can act on. + Each Webhook can be limited to firing only on certain actions or certain object types. + """ + name = models.CharField( + verbose_name=_('name'), + max_length=150, + unique=True + ) payload_url = models.CharField( max_length=500, verbose_name=_('URL'), @@ -86,10 +190,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "processing is supported with the same context as the request body." ) ) - enabled = models.BooleanField( - verbose_name=_('enabled'), - default=True - ) http_method = models.CharField( max_length=30, choices=WebhookHttpMethodChoices, @@ -132,12 +232,6 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "digest of the payload body using the secret as the key. The secret is not transmitted in the request." ) ) - conditions = models.JSONField( - verbose_name=_('conditions'), - blank=True, - null=True, - help_text=_("A set of conditions which determine whether the webhook will be generated.") - ) ssl_verification = models.BooleanField( default=True, verbose_name=_('SSL verification'), @@ -152,15 +246,14 @@ class Webhook(CustomFieldsMixin, ExportTemplatesMixin, TagsMixin, ChangeLoggedMo "The specific CA certificate file to use for SSL verification. Leave blank to use the system defaults." ) ) + events = GenericRelation( + EventRule, + content_type_field='action_object_type', + object_id_field='action_object_id' + ) class Meta: ordering = ('name',) - constraints = ( - models.UniqueConstraint( - fields=('payload_url', 'type_create', 'type_update', 'type_delete'), - name='%(app_label)s_%(class)s_unique_payload_url_types' - ), - ) verbose_name = _('webhook') verbose_name_plural = _('webhooks') @@ -177,20 +270,6 @@ def docs_url(self): def clean(self): super().clean() - # At least one action type must be selected - if not any([ - self.type_create, self.type_update, self.type_delete, self.type_job_start, self.type_job_end - ]): - raise ValidationError( - _("At least one event type must be selected: create, update, delete, job_start, and/or job_end.") - ) - - if self.conditions: - try: - ConditionSet(self.conditions) - except ValueError as e: - raise ValidationError({'conditions': e}) - # CA file path requires SSL verification enabled if not self.ssl_verification and self.ca_file_path: raise ValidationError({ diff --git a/netbox/extras/models/reports.py b/netbox/extras/models/reports.py index 223d679bd5f..f6228ef24a7 100644 --- a/netbox/extras/models/reports.py +++ b/netbox/extras/models/reports.py @@ -9,7 +9,7 @@ from core.choices import ManagedFileRootPathChoices from core.models import ManagedFile from extras.utils import is_report -from netbox.models.features import JobsMixin, WebhooksMixin +from netbox.models.features import JobsMixin, EventRulesMixin from utilities.querysets import RestrictedQuerySet from .mixins import PythonModuleMixin @@ -21,7 +21,7 @@ ) -class Report(WebhooksMixin, models.Model): +class Report(EventRulesMixin, models.Model): """ Dummy model used to generate permissions for reports. Does not exist in the database. """ diff --git a/netbox/extras/models/scripts.py b/netbox/extras/models/scripts.py index 122f56f2065..93275acdab0 100644 --- a/netbox/extras/models/scripts.py +++ b/netbox/extras/models/scripts.py @@ -9,7 +9,7 @@ from core.choices import ManagedFileRootPathChoices from core.models import ManagedFile from extras.utils import is_script -from netbox.models.features import JobsMixin, WebhooksMixin +from netbox.models.features import JobsMixin, EventRulesMixin from utilities.querysets import RestrictedQuerySet from .mixins import PythonModuleMixin @@ -21,7 +21,7 @@ logger = logging.getLogger('netbox.data_backends') -class Script(WebhooksMixin, models.Model): +class Script(EventRulesMixin, models.Model): """ Dummy model used to generate permissions for custom scripts. Does not exist in the database. """ diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index df75200e696..495957fd97b 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -17,13 +17,13 @@ from extras.api.serializers import ScriptOutputSerializer from extras.choices import LogLevelChoices from extras.models import ScriptModule -from extras.signals import clear_webhooks +from extras.signals import clear_events from ipam.formfields import IPAddressFormField, IPNetworkFormField from ipam.validators import MaxPrefixLengthValidator, MinPrefixLengthValidator, prefix_validator from utilities.exceptions import AbortScript, AbortTransaction from utilities.forms import add_blank_choice from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField -from .context_managers import change_logging +from .context_managers import event_tracking from .forms import ScriptForm __all__ = ( @@ -472,10 +472,16 @@ def get_module_and_script(module_name, script_name): return module, script -def run_script(data, request, job, commit=True, **kwargs): +def run_script(data, job, request=None, commit=True, **kwargs): """ A wrapper for calling Script.run(). This performs error handling and provides a hook for committing changes. It exists outside the Script class to ensure it cannot be overridden by a script author. + + Args: + data: A dictionary of data to be passed to the script upon execution + job: The Job associated with this execution + request: The WSGI request associated with this execution (if any) + commit: Passed through to Script.run() """ job.start() @@ -486,9 +492,10 @@ def run_script(data, request, job, commit=True, **kwargs): logger.info(f"Running script (commit={commit})") # Add files to form data - files = request.FILES - for field_name, fileobj in files.items(): - data[field_name] = fileobj + if request: + files = request.FILES + for field_name, fileobj in files.items(): + data[field_name] = fileobj # Add the current request as a property of the script script.request = request @@ -496,7 +503,7 @@ def run_script(data, request, job, commit=True, **kwargs): def _run_script(): """ Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with - the change_logging context manager (which is bypassed if commit == False). + the event_tracking context manager (which is bypassed if commit == False). """ try: try: @@ -506,7 +513,8 @@ def _run_script(): raise AbortTransaction() except AbortTransaction: script.log_info("Database changes have been reverted automatically.") - clear_webhooks.send(request) + if request: + clear_events.send(request) job.data = ScriptOutputSerializer(script).data job.terminate() except Exception as e: @@ -520,14 +528,15 @@ def _run_script(): script.log_info("Database changes have been reverted due to error.") job.data = ScriptOutputSerializer(script).data job.terminate(status=JobStatusChoices.STATUS_ERRORED, error=str(e)) - clear_webhooks.send(request) + if request: + clear_events.send(request) logger.info(f"Script completed in {job.duration}") - # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process - # change logging, webhooks, etc. + # Execute the script. If commit is True, wrap it with the event_tracking context manager to ensure we process + # change logging, event rules, etc. if commit: - with change_logging(request): + with event_tracking(request): _run_script() else: _run_script() diff --git a/netbox/extras/signals.py b/netbox/extras/signals.py index e1d424960bd..184ee6d9b89 100644 --- a/netbox/extras/signals.py +++ b/netbox/extras/signals.py @@ -10,19 +10,19 @@ from extras.validators import CustomValidator from netbox.config import get_config -from netbox.context import current_request, webhooks_queue +from netbox.context import current_request, events_queue from netbox.signals import post_clean from utilities.exceptions import AbortRequest from .choices import ObjectChangeActionChoices +from .events import enqueue_object, get_snapshots, serialize_for_event from .models import CustomField, ObjectChange, TaggedItem -from .webhooks import enqueue_object, get_snapshots, serialize_for_webhook # # Change logging/webhooks # -# Define a custom signal that can be sent to clear any queued webhooks -clear_webhooks = Signal() +# Define a custom signal that can be sent to clear any queued events +clear_events = Signal() def is_same_object(instance, webhook_data, request_id): @@ -81,14 +81,14 @@ def handle_changed_object(sender, instance, **kwargs): objectchange.save() # If this is an M2M change, update the previously queued webhook (from post_save) - queue = webhooks_queue.get() + queue = events_queue.get() if m2m_changed and queue and is_same_object(instance, queue[-1], request.id): instance.refresh_from_db() # Ensure that we're working with fresh M2M assignments - queue[-1]['data'] = serialize_for_webhook(instance) + queue[-1]['data'] = serialize_for_event(instance) queue[-1]['snapshots']['postchange'] = get_snapshots(instance, action)['postchange'] else: enqueue_object(queue, instance, request.user, request.id, action) - webhooks_queue.set(queue) + events_queue.set(queue) # Increment metric counters if action == ObjectChangeActionChoices.ACTION_CREATE: @@ -117,22 +117,22 @@ def handle_deleted_object(sender, instance, **kwargs): objectchange.save() # Enqueue webhooks - queue = webhooks_queue.get() + queue = events_queue.get() enqueue_object(queue, instance, request.user, request.id, ObjectChangeActionChoices.ACTION_DELETE) - webhooks_queue.set(queue) + events_queue.set(queue) # Increment metric counters model_deletes.labels(instance._meta.model_name).inc() -@receiver(clear_webhooks) -def clear_webhook_queue(sender, **kwargs): +@receiver(clear_events) +def clear_events_queue(sender, **kwargs): """ - Delete any queued webhooks (e.g. because of an aborted bulk transaction) + Delete any queued events (e.g. because of an aborted bulk transaction) """ - logger = logging.getLogger('webhooks') - logger.info(f"Clearing {len(webhooks_queue.get())} queued webhooks ({sender})") - webhooks_queue.set([]) + logger = logging.getLogger('events') + logger.info(f"Clearing {len(events_queue.get())} queued events ({sender})") + events_queue.set([]) # diff --git a/netbox/extras/tables/tables.py b/netbox/extras/tables/tables.py index b78ab0c94ff..ece23093b00 100644 --- a/netbox/extras/tables/tables.py +++ b/netbox/extras/tables/tables.py @@ -15,6 +15,7 @@ 'CustomFieldChoiceSetTable', 'CustomFieldTable', 'CustomLinkTable', + 'EventRuleTable', 'ExportTemplateTable', 'ImageAttachmentTable', 'JournalEntryTable', @@ -250,6 +251,32 @@ class WebhookTable(NetBoxTable): verbose_name=_('Name'), linkify=True ) + ssl_validation = columns.BooleanColumn( + verbose_name=_('SSL Validation') + ) + tags = columns.TagColumn( + url_name='extras:webhook_list' + ) + + class Meta(NetBoxTable.Meta): + model = Webhook + fields = ( + 'pk', 'id', 'name', 'http_method', 'payload_url', 'http_content_type', 'secret', 'ssl_verification', + 'ca_file_path', 'tags', 'created', 'last_updated', + ) + default_columns = ( + 'pk', 'name', 'http_method', 'payload_url', + ) + + +class EventRuleTable(NetBoxTable): + name = tables.Column( + verbose_name=_('Name'), + linkify=True + ) + action_type = tables.Column( + verbose_name=_('Action Type'), + ) content_types = columns.ContentTypesColumn( verbose_name=_('Content Types'), ) @@ -271,23 +298,19 @@ class WebhookTable(NetBoxTable): type_job_end = columns.BooleanColumn( verbose_name=_('Job End') ) - ssl_validation = columns.BooleanColumn( - verbose_name=_('SSL Validation') - ) tags = columns.TagColumn( url_name='extras:webhook_list' ) class Meta(NetBoxTable.Meta): - model = Webhook + model = EventRule fields = ( - 'pk', 'id', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', - 'type_job_start', 'type_job_end', 'http_method', 'payload_url', 'secret', 'ssl_validation', 'ca_file_path', - 'tags', 'created', 'last_updated', + 'pk', 'id', 'name', 'enabled', 'description', 'action_type', 'content_types', 'type_create', 'type_update', + 'type_delete', 'type_job_start', 'type_job_end', 'tags', 'created', 'last_updated', ) default_columns = ( - 'pk', 'name', 'content_types', 'enabled', 'type_create', 'type_update', 'type_delete', 'type_job_start', - 'type_job_end', 'http_method', 'payload_url', + 'pk', 'name', 'enabled', 'action_type', 'content_types', 'type_create', 'type_update', 'type_delete', + 'type_job_start', 'type_job_end', ) diff --git a/netbox/extras/tests/test_api.py b/netbox/extras/tests/test_api.py index 255457f21b9..b35fb8d6615 100644 --- a/netbox/extras/tests/test_api.py +++ b/netbox/extras/tests/test_api.py @@ -8,6 +8,7 @@ from core.choices import ManagedFileRootPathChoices from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Rack, Location, RackRole, Site +from extras.choices import * from extras.models import * from extras.reports import Report from extras.scripts import BooleanVar, IntegerVar, Script, StringVar @@ -32,21 +33,15 @@ class WebhookTest(APIViewTestCases.APIViewTestCase): brief_fields = ['display', 'id', 'name', 'url'] create_data = [ { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 4', - 'type_create': True, 'payload_url': 'http://example.com/?4', }, { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 5', - 'type_update': True, 'payload_url': 'http://example.com/?5', }, { - 'content_types': ['dcim.device', 'dcim.devicetype'], 'name': 'Webhook 6', - 'type_delete': True, 'payload_url': 'http://example.com/?6', }, ] @@ -56,29 +51,100 @@ class WebhookTest(APIViewTestCases.APIViewTestCase): @classmethod def setUpTestData(cls): - site_ct = ContentType.objects.get_for_model(Site) - rack_ct = ContentType.objects.get_for_model(Rack) webhooks = ( Webhook( name='Webhook 1', - type_create=True, payload_url='http://example.com/?1', ), Webhook( name='Webhook 2', - type_update=True, payload_url='http://example.com/?1', ), Webhook( name='Webhook 3', - type_delete=True, payload_url='http://example.com/?1', ), ) Webhook.objects.bulk_create(webhooks) - for webhook in webhooks: - webhook.content_types.add(site_ct, rack_ct) + + +class EventRuleTest(APIViewTestCases.APIViewTestCase): + model = EventRule + brief_fields = ['display', 'id', 'name', 'url'] + bulk_update_data = { + 'enabled': False, + 'description': 'New description', + } + update_data = { + 'name': 'Event Rule X', + 'enabled': False, + 'description': 'New description', + } + + @classmethod + def setUpTestData(cls): + webhooks = ( + Webhook( + name='Webhook 1', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 2', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 3', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 4', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 5', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 6', + payload_url='http://example.com/?1', + ), + ) + Webhook.objects.bulk_create(webhooks) + + event_rules = ( + EventRule(name='EventRule 1', type_create=True, action_object=webhooks[0]), + EventRule(name='EventRule 2', type_create=True, action_object=webhooks[1]), + EventRule(name='EventRule 3', type_create=True, action_object=webhooks[2]), + ) + EventRule.objects.bulk_create(event_rules) + + cls.create_data = [ + { + 'name': 'EventRule 4', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[3].pk, + }, + { + 'name': 'EventRule 5', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[4].pk, + }, + { + 'name': 'EventRule 6', + 'content_types': ['dcim.device', 'dcim.devicetype'], + 'type_create': True, + 'action_type': EventRuleActionChoices.WEBHOOK, + 'action_object_type': 'extras.webhook', + 'action_object_id': webhooks[5].pk, + }, + ] class CustomFieldTest(APIViewTestCases.APIViewTestCase): diff --git a/netbox/extras/tests/test_webhooks.py b/netbox/extras/tests/test_event_rules.py similarity index 72% rename from netbox/extras/tests/test_webhooks.py rename to netbox/extras/tests/test_event_rules.py index ef76377652d..ed64ba89144 100644 --- a/netbox/extras/tests/test_webhooks.py +++ b/netbox/extras/tests/test_event_rules.py @@ -3,22 +3,22 @@ from unittest.mock import patch import django_rq +from dcim.choices import SiteStatusChoices +from dcim.models import Site from django.contrib.contenttypes.models import ContentType from django.http import HttpResponse from django.urls import reverse +from extras.choices import EventRuleActionChoices, ObjectChangeActionChoices +from extras.events import enqueue_object, flush_events, serialize_for_event +from extras.models import EventRule, Tag, Webhook +from extras.webhooks import generate_signature +from extras.webhooks_worker import process_webhook from requests import Session from rest_framework import status - -from dcim.choices import SiteStatusChoices -from dcim.models import Site -from extras.choices import ObjectChangeActionChoices -from extras.models import Tag, Webhook -from extras.webhooks import enqueue_object, flush_webhooks, generate_signature, serialize_for_webhook -from extras.webhooks_worker import eval_conditions, process_webhook from utilities.testing import APITestCase -class WebhookTest(APITestCase): +class EventRuleTest(APITestCase): def setUp(self): super().setUp() @@ -35,12 +35,37 @@ def setUpTestData(cls): DUMMY_SECRET = 'LOOKATMEIMASECRETSTRING' webhooks = Webhook.objects.bulk_create(( - Webhook(name='Webhook 1', type_create=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), - Webhook(name='Webhook 2', type_update=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), - Webhook(name='Webhook 3', type_delete=True, payload_url=DUMMY_URL, secret=DUMMY_SECRET), + Webhook(name='Webhook 1', payload_url=DUMMY_URL, secret=DUMMY_SECRET, additional_headers='X-Foo: Bar'), + Webhook(name='Webhook 2', payload_url=DUMMY_URL, secret=DUMMY_SECRET), + Webhook(name='Webhook 3', payload_url=DUMMY_URL, secret=DUMMY_SECRET), + )) + + ct = ContentType.objects.get(app_label='extras', model='webhook') + event_rules = EventRule.objects.bulk_create(( + EventRule( + name='Webhook Event 1', + type_create=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + EventRule( + name='Webhook Event 2', + type_update=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), + EventRule( + name='Webhook Event 3', + type_delete=True, + action_type=EventRuleActionChoices.WEBHOOK, + action_object_type=ct, + action_object_id=webhooks[0].id + ), )) - for webhook in webhooks: - webhook.content_types.set([site_ct]) + for event_rule in event_rules: + event_rule.content_types.set([site_ct]) Tag.objects.bulk_create(( Tag(name='Foo', slug='foo'), @@ -48,7 +73,42 @@ def setUpTestData(cls): Tag(name='Baz', slug='baz'), )) - def test_enqueue_webhook_create(self): + def test_eventrule_conditions(self): + """ + Test evaluation of EventRule conditions. + """ + event_rule = EventRule( + name='Event Rule 1', + type_create=True, + type_update=True, + conditions={ + 'and': [ + { + 'attr': 'status.value', + 'value': 'active', + } + ] + } + ) + + # Create a Site to evaluate + site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) + data = serialize_for_event(site) + + # Evaluate the conditions (status='staging') + self.assertFalse(event_rule.eval_conditions(data)) + + # Change the site's status + site.status = SiteStatusChoices.STATUS_ACTIVE + data = serialize_for_event(site) + + # Evaluate the conditions (status='active') + self.assertTrue(event_rule.eval_conditions(data)) + + def test_single_create_process_eventrule(self): + """ + Check that creating an object with an applicable EventRule queues a background task for the rule's action. + """ # Create an object via the REST API data = { 'name': 'Site 1', @@ -65,10 +125,10 @@ def test_enqueue_webhook_create(self): self.assertEqual(Site.objects.count(), 1) self.assertEqual(Site.objects.first().tags.count(), 2) - # Verify that a job was queued for the object creation webhook + # Verify that a background task was queued for the new object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], response.data['id']) @@ -76,7 +136,11 @@ def test_enqueue_webhook_create(self): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site 1') self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_bulk_create(self): + def test_bulk_create_process_eventrule(self): + """ + Check that bulk creating multiple objects with an applicable EventRule queues a background task for each + new object. + """ # Create multiple objects via the REST API data = [ { @@ -111,10 +175,10 @@ def test_enqueue_webhook_bulk_create(self): self.assertEqual(Site.objects.count(), 3) self.assertEqual(Site.objects.first().tags.count(), 2) - # Verify that a webhook was queued for each object + # Verify that a background task was queued for each new object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_create=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_create=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_CREATE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], response.data[i]['id']) @@ -122,7 +186,10 @@ def test_enqueue_webhook_bulk_create(self): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_update(self): + def test_single_update_process_eventrule(self): + """ + Check that updating an object with an applicable EventRule queues a background task for the rule's action. + """ site = Site.objects.create(name='Site 1', slug='site-1') site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) @@ -139,10 +206,10 @@ def test_enqueue_webhook_update(self): response = self.client.patch(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for the updated object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], site.pk) @@ -152,7 +219,11 @@ def test_enqueue_webhook_update(self): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], 'Site X') self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - def test_enqueue_webhook_bulk_update(self): + def test_bulk_update_process_eventrule(self): + """ + Check that bulk updating multiple objects with an applicable EventRule queues a background task for each + updated object. + """ sites = ( Site(name='Site 1', slug='site-1'), Site(name='Site 2', slug='site-2'), @@ -191,10 +262,10 @@ def test_enqueue_webhook_bulk_update(self): response = self.client.patch(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_200_OK) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for each updated object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_update=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_update=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_UPDATE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], data[i]['id']) @@ -204,7 +275,10 @@ def test_enqueue_webhook_bulk_update(self): self.assertEqual(job.kwargs['snapshots']['postchange']['name'], response.data[i]['name']) self.assertEqual(job.kwargs['snapshots']['postchange']['tags'], ['Baz']) - def test_enqueue_webhook_delete(self): + def test_single_delete_process_eventrule(self): + """ + Check that deleting an object with an applicable EventRule queues a background task for the rule's action. + """ site = Site.objects.create(name='Site 1', slug='site-1') site.tags.set(Tag.objects.filter(name__in=['Foo', 'Bar'])) @@ -214,17 +288,21 @@ def test_enqueue_webhook_delete(self): response = self.client.delete(url, **self.header) self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - # Verify that a job was queued for the object update webhook + # Verify that a task was queued for the deleted object self.assertEqual(self.queue.count, 1) job = self.queue.jobs[0] - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], site.pk) self.assertEqual(job.kwargs['snapshots']['prechange']['name'], 'Site 1') self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - def test_enqueue_webhook_bulk_delete(self): + def test_bulk_delete_process_eventrule(self): + """ + Check that bulk deleting multiple objects with an applicable EventRule queues a background task for each + deleted object. + """ sites = ( Site(name='Site 1', slug='site-1'), Site(name='Site 2', slug='site-2'), @@ -243,49 +321,17 @@ def test_enqueue_webhook_bulk_delete(self): response = self.client.delete(url, data, format='json', **self.header) self.assertHttpStatus(response, status.HTTP_204_NO_CONTENT) - # Verify that a job was queued for the object update webhook + # Verify that a background task was queued for each deleted object self.assertEqual(self.queue.count, 3) for i, job in enumerate(self.queue.jobs): - self.assertEqual(job.kwargs['webhook'], Webhook.objects.get(type_delete=True)) + self.assertEqual(job.kwargs['event_rule'], EventRule.objects.get(type_delete=True)) self.assertEqual(job.kwargs['event'], ObjectChangeActionChoices.ACTION_DELETE) self.assertEqual(job.kwargs['model_name'], 'site') self.assertEqual(job.kwargs['data']['id'], sites[i].pk) self.assertEqual(job.kwargs['snapshots']['prechange']['name'], sites[i].name) self.assertEqual(job.kwargs['snapshots']['prechange']['tags'], ['Bar', 'Foo']) - def test_webhook_conditions(self): - # Create a conditional Webhook - webhook = Webhook( - name='Conditional Webhook', - type_create=True, - type_update=True, - payload_url='http://localhost:9000/', - conditions={ - 'and': [ - { - 'attr': 'status.value', - 'value': 'active', - } - ] - } - ) - - # Create a Site to evaluate - site = Site.objects.create(name='Site 1', slug='site-1', status=SiteStatusChoices.STATUS_STAGING) - data = serialize_for_webhook(site) - - # Evaluate the conditions (status='staging') - self.assertFalse(eval_conditions(webhook, data)) - - # Change the site's status - site.status = SiteStatusChoices.STATUS_ACTIVE - data = serialize_for_webhook(site) - - # Evaluate the conditions (status='active') - self.assertTrue(eval_conditions(webhook, data)) - def test_webhooks_worker(self): - request_id = uuid.uuid4() def dummy_send(_, request, **kwargs): @@ -293,7 +339,8 @@ def dummy_send(_, request, **kwargs): A dummy implementation of Session.send() to be used for testing. Always returns a 200 HTTP response. """ - webhook = Webhook.objects.get(type_create=True) + event = EventRule.objects.get(type_create=True) + webhook = event.action_object signature = generate_signature(request.body, webhook.secret) # Validate the outgoing request headers @@ -322,7 +369,7 @@ def dummy_send(_, request, **kwargs): request_id=request_id, action=ObjectChangeActionChoices.ACTION_CREATE ) - flush_webhooks(webhooks_queue) + flush_events(webhooks_queue) # Retrieve the job from queue job = self.queue.jobs[0] diff --git a/netbox/extras/tests/test_filtersets.py b/netbox/extras/tests/test_filtersets.py index c5a6706c07f..ddc5feb40ad 100644 --- a/netbox/extras/tests/test_filtersets.py +++ b/netbox/extras/tests/test_filtersets.py @@ -6,6 +6,7 @@ from django.test import TestCase from circuits.models import Provider +from core.choices import ManagedFileRootPathChoices from dcim.filtersets import SiteFilterSet from dcim.models import DeviceRole, DeviceType, Manufacturer, Platform, Rack, Region, Site, SiteGroup from dcim.models import Location @@ -159,82 +160,174 @@ def setUpTestData(cls): webhooks = ( Webhook( name='Webhook 1', - type_create=True, - type_update=False, - type_delete=False, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?1', - enabled=True, http_method='GET', ssl_verification=True, ), Webhook( name='Webhook 2', - type_create=False, - type_update=True, - type_delete=False, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?2', - enabled=True, http_method='POST', ssl_verification=True, ), Webhook( name='Webhook 3', - type_create=False, - type_update=False, - type_delete=True, - type_job_start=False, - type_job_end=False, payload_url='http://example.com/?3', - enabled=False, http_method='PATCH', ssl_verification=False, ), Webhook( name='Webhook 4', - type_create=False, - type_update=False, - type_delete=False, - type_job_start=True, - type_job_end=False, payload_url='http://example.com/?4', - enabled=False, http_method='PATCH', ssl_verification=False, ), Webhook( name='Webhook 5', - type_create=False, - type_update=False, - type_delete=False, - type_job_start=False, - type_job_end=True, payload_url='http://example.com/?5', - enabled=False, http_method='PATCH', ssl_verification=False, ), ) Webhook.objects.bulk_create(webhooks) - webhooks[0].content_types.add(content_types[0]) - webhooks[1].content_types.add(content_types[1]) - webhooks[2].content_types.add(content_types[2]) - webhooks[3].content_types.add(content_types[3]) - webhooks[4].content_types.add(content_types[4]) def test_name(self): params = {'name': ['Webhook 1', 'Webhook 2']} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_http_method(self): + params = {'http_method': ['GET', 'POST']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_ssl_verification(self): + params = {'ssl_verification': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + +class EventRuleTestCase(TestCase, BaseFilterSetTests): + queryset = EventRule.objects.all() + filterset = EventRuleFilterSet + + @classmethod + def setUpTestData(cls): + content_types = ContentType.objects.filter( + model__in=['region', 'site', 'rack', 'location', 'device'] + ) + + webhooks = ( + Webhook( + name='Webhook 1', + payload_url='http://example.com/?1', + ), + Webhook( + name='Webhook 2', + payload_url='http://example.com/?2', + ), + Webhook( + name='Webhook 3', + payload_url='http://example.com/?3', + ), + ) + Webhook.objects.bulk_create(webhooks) + + scripts = ( + ScriptModule( + file_root=ManagedFileRootPathChoices.SCRIPTS, + file_path='/var/tmp/script1.py' + ), + ScriptModule( + file_root=ManagedFileRootPathChoices.SCRIPTS, + file_path='/var/tmp/script2.py' + ), + ) + ScriptModule.objects.bulk_create(scripts) + + event_rules = ( + EventRule( + name='Event Rule 1', + action_object=webhooks[0], + enabled=True, + type_create=True, + type_update=False, + type_delete=False, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='Event Rule 2', + action_object=webhooks[1], + enabled=True, + type_create=False, + type_update=True, + type_delete=False, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='Event Rule 3', + action_object=webhooks[2], + enabled=False, + type_create=False, + type_update=False, + type_delete=True, + type_job_start=False, + type_job_end=False, + action_type=EventRuleActionChoices.WEBHOOK, + ), + EventRule( + name='Event Rule 4', + action_object=scripts[0], + enabled=False, + type_create=False, + type_update=False, + type_delete=False, + type_job_start=True, + type_job_end=False, + action_type=EventRuleActionChoices.SCRIPT, + ), + EventRule( + name='Event Rule 5', + action_object=scripts[1], + enabled=False, + type_create=False, + type_update=False, + type_delete=False, + type_job_start=False, + type_job_end=True, + action_type=EventRuleActionChoices.SCRIPT, + ), + ) + EventRule.objects.bulk_create(event_rules) + event_rules[0].content_types.add(content_types[0]) + event_rules[1].content_types.add(content_types[1]) + event_rules[2].content_types.add(content_types[2]) + event_rules[3].content_types.add(content_types[3]) + event_rules[4].content_types.add(content_types[4]) + + def test_name(self): + params = {'name': ['Event Rule 1', 'Event Rule 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + def test_content_types(self): params = {'content_types': 'dcim.region'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) params = {'content_type_id': [ContentType.objects.get_for_model(Region).pk]} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + def test_action_type(self): + params = {'action_type': [EventRuleActionChoices.WEBHOOK]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + params = {'action_type': [EventRuleActionChoices.SCRIPT]} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_enabled(self): + params = {'enabled': True} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + params = {'enabled': False} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3) + def test_type_create(self): params = {'type_create': True} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) @@ -255,18 +348,6 @@ def test_type_job_end(self): params = {'type_job_end': True} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) - def test_enabled(self): - params = {'enabled': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) - - def test_http_method(self): - params = {'http_method': ['GET', 'POST']} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) - - def test_ssl_verification(self): - params = {'ssl_verification': True} - self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) - class CustomLinkTestCase(TestCase, BaseFilterSetTests): queryset = CustomLink.objects.all() diff --git a/netbox/extras/tests/test_views.py b/netbox/extras/tests/test_views.py index 3d4b3e9a9f2..602a9d4deb8 100644 --- a/netbox/extras/tests/test_views.py +++ b/netbox/extras/tests/test_views.py @@ -1,4 +1,3 @@ -import json import urllib.parse import uuid @@ -11,7 +10,6 @@ from extras.models import * from utilities.testing import ViewTestCases, TestCase - User = get_user_model() @@ -336,33 +334,26 @@ class WebhookTestCase(ViewTestCases.PrimaryObjectViewTestCase): @classmethod def setUpTestData(cls): - site_ct = ContentType.objects.get_for_model(Site) webhooks = ( - Webhook(name='Webhook 1', payload_url='http://example.com/?1', type_create=True, http_method='POST'), - Webhook(name='Webhook 2', payload_url='http://example.com/?2', type_create=True, http_method='POST'), - Webhook(name='Webhook 3', payload_url='http://example.com/?3', type_create=True, http_method='POST'), + Webhook(name='Webhook 1', payload_url='http://example.com/?1', http_method='POST'), + Webhook(name='Webhook 2', payload_url='http://example.com/?2', http_method='POST'), + Webhook(name='Webhook 3', payload_url='http://example.com/?3', http_method='POST'), ) for webhook in webhooks: webhook.save() - webhook.content_types.add(site_ct) cls.form_data = { 'name': 'Webhook X', - 'content_types': [site_ct.pk], - 'type_create': False, - 'type_update': True, - 'type_delete': True, 'payload_url': 'http://example.com/?x', 'http_method': 'GET', 'http_content_type': 'application/foo', - 'conditions': None, } cls.csv_data = ( - "name,content_types,type_create,payload_url,http_method,http_content_type", - "Webhook 4,dcim.site,True,http://example.com/?4,GET,application/json", - "Webhook 5,dcim.site,True,http://example.com/?5,GET,application/json", - "Webhook 6,dcim.site,True,http://example.com/?6,GET,application/json", + "name,payload_url,http_method,http_content_type", + "Webhook 4,http://example.com/?4,GET,application/json", + "Webhook 5,http://example.com/?5,GET,application/json", + "Webhook 6,http://example.com/?6,GET,application/json", ) cls.csv_update_data = ( @@ -373,11 +364,62 @@ def setUpTestData(cls): ) cls.bulk_edit_data = { - 'enabled': False, + 'http_method': 'GET', + } + + +class EventRulesTestCase(ViewTestCases.PrimaryObjectViewTestCase): + model = EventRule + + @classmethod + def setUpTestData(cls): + + webhooks = ( + Webhook(name='Webhook 1', payload_url='http://example.com/?1', http_method='POST'), + Webhook(name='Webhook 2', payload_url='http://example.com/?2', http_method='POST'), + Webhook(name='Webhook 3', payload_url='http://example.com/?3', http_method='POST'), + ) + for webhook in webhooks: + webhook.save() + + site_ct = ContentType.objects.get_for_model(Site) + event_rules = ( + EventRule(name='EventRule 1', type_create=True, action_object=webhooks[0]), + EventRule(name='EventRule 2', type_create=True, action_object=webhooks[1]), + EventRule(name='EventRule 3', type_create=True, action_object=webhooks[2]), + ) + for event in event_rules: + event.save() + event.content_types.add(site_ct) + + webhook_ct = ContentType.objects.get_for_model(Webhook) + cls.form_data = { + 'name': 'Event X', + 'content_types': [site_ct.pk], 'type_create': False, 'type_update': True, 'type_delete': True, - 'http_method': 'GET', + 'conditions': None, + 'action_type': 'webhook', + 'action_object_type': webhook_ct.pk, + 'action_object_id': webhooks[0].pk, + 'action_choice': webhooks[0] + } + + cls.csv_data = ( + "name,content_types,type_create,action_type,action_object", + "Webhook 4,dcim.site,True,webhook,Webhook 1", + ) + + cls.csv_update_data = ( + "id,name", + f"{event_rules[0].pk},Event 7", + f"{event_rules[1].pk},Event 8", + f"{event_rules[2].pk},Event 9", + ) + + cls.bulk_edit_data = { + 'type_update': True, } diff --git a/netbox/extras/urls.py b/netbox/extras/urls.py index bcab007e75a..0a1786f1f39 100644 --- a/netbox/extras/urls.py +++ b/netbox/extras/urls.py @@ -61,6 +61,14 @@ path('webhooks/delete/', views.WebhookBulkDeleteView.as_view(), name='webhook_bulk_delete'), path('webhooks//', include(get_model_urls('extras', 'webhook'))), + # Event rules + path('event-rules/', views.EventRuleListView.as_view(), name='eventrule_list'), + path('event-rules/add/', views.EventRuleEditView.as_view(), name='eventrule_add'), + path('event-rules/import/', views.EventRuleBulkImportView.as_view(), name='eventrule_import'), + path('event-rules/edit/', views.EventRuleBulkEditView.as_view(), name='eventrule_bulk_edit'), + path('event-rules/delete/', views.EventRuleBulkDeleteView.as_view(), name='eventrule_bulk_delete'), + path('event-rules//', include(get_model_urls('extras', 'eventrule'))), + # Tags path('tags/', views.TagListView.as_view(), name='tag_list'), path('tags/add/', views.TagEditView.as_view(), name='tag_add'), diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 97aed673aa4..a3dd7f193cf 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -395,6 +395,51 @@ class WebhookBulkDeleteView(generic.BulkDeleteView): table = tables.WebhookTable +# +# Event Rules +# + +class EventRuleListView(generic.ObjectListView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + filterset_form = forms.EventRuleFilterForm + table = tables.EventRuleTable + + +@register_model_view(EventRule) +class EventRuleView(generic.ObjectView): + queryset = EventRule.objects.all() + + +@register_model_view(EventRule, 'edit') +class EventRuleEditView(generic.ObjectEditView): + queryset = EventRule.objects.all() + form = forms.EventRuleForm + + +@register_model_view(EventRule, 'delete') +class EventRuleDeleteView(generic.ObjectDeleteView): + queryset = EventRule.objects.all() + + +class EventRuleBulkImportView(generic.BulkImportView): + queryset = EventRule.objects.all() + model_form = forms.EventRuleImportForm + + +class EventRuleBulkEditView(generic.BulkEditView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + table = tables.EventRuleTable + form = forms.EventRuleBulkEditForm + + +class EventRuleBulkDeleteView(generic.BulkDeleteView): + queryset = EventRule.objects.all() + filterset = filtersets.EventRuleFilterSet + table = tables.EventRuleTable + + # # Tags # diff --git a/netbox/extras/webhooks.py b/netbox/extras/webhooks.py index a22f73c27e4..a48a8038b3d 100644 --- a/netbox/extras/webhooks.py +++ b/netbox/extras/webhooks.py @@ -1,47 +1,6 @@ import hashlib import hmac -from django.contrib.contenttypes.models import ContentType -from django.utils import timezone -from django_rq import get_queue - -from netbox.config import get_config -from netbox.constants import RQ_QUEUE_DEFAULT -from netbox.registry import registry -from utilities.api import get_serializer_for_model -from utilities.rqworker import get_rq_retry -from utilities.utils import serialize_object -from .choices import * -from .models import Webhook - - -def serialize_for_webhook(instance): - """ - Return a serialized representation of the given instance suitable for use in a webhook. - """ - serializer_class = get_serializer_for_model(instance.__class__) - serializer_context = { - 'request': None, - } - serializer = serializer_class(instance, context=serializer_context) - - return serializer.data - - -def get_snapshots(instance, action): - snapshots = { - 'prechange': getattr(instance, '_prechange_snapshot', None), - 'postchange': None, - } - if action != ObjectChangeActionChoices.ACTION_DELETE: - # Use model's serialize_object() method if defined; fall back to serialize_object() utility function - if hasattr(instance, 'serialize_object'): - snapshots['postchange'] = instance.serialize_object() - else: - snapshots['postchange'] = serialize_object(instance) - - return snapshots - def generate_signature(request_body, secret): """ @@ -53,70 +12,3 @@ def generate_signature(request_body, secret): digestmod=hashlib.sha512 ) return hmac_prep.hexdigest() - - -def enqueue_object(queue, instance, user, request_id, action): - """ - Enqueue a serialized representation of a created/updated/deleted object for the processing of - webhooks once the request has completed. - """ - # Determine whether this type of object supports webhooks - app_label = instance._meta.app_label - model_name = instance._meta.model_name - if model_name not in registry['model_features']['webhooks'].get(app_label, []): - return - - queue.append({ - 'content_type': ContentType.objects.get_for_model(instance), - 'object_id': instance.pk, - 'event': action, - 'data': serialize_for_webhook(instance), - 'snapshots': get_snapshots(instance, action), - 'username': user.username, - 'request_id': request_id - }) - - -def flush_webhooks(queue): - """ - Flush a list of object representation to RQ for webhook processing. - """ - rq_queue_name = get_config().QUEUE_MAPPINGS.get('webhook', RQ_QUEUE_DEFAULT) - rq_queue = get_queue(rq_queue_name) - webhooks_cache = { - 'type_create': {}, - 'type_update': {}, - 'type_delete': {}, - } - - for data in queue: - - action_flag = { - ObjectChangeActionChoices.ACTION_CREATE: 'type_create', - ObjectChangeActionChoices.ACTION_UPDATE: 'type_update', - ObjectChangeActionChoices.ACTION_DELETE: 'type_delete', - }[data['event']] - content_type = data['content_type'] - - # Cache applicable Webhooks - if content_type not in webhooks_cache[action_flag]: - webhooks_cache[action_flag][content_type] = Webhook.objects.filter( - **{action_flag: True}, - content_types=content_type, - enabled=True - ) - webhooks = webhooks_cache[action_flag][content_type] - - for webhook in webhooks: - rq_queue.enqueue( - "extras.webhooks_worker.process_webhook", - webhook=webhook, - model_name=content_type.model, - event=data['event'], - data=data['data'], - snapshots=data['snapshots'], - timestamp=timezone.now().isoformat(), - username=data['username'], - request_id=data['request_id'], - retry=get_rq_retry() - ) diff --git a/netbox/extras/webhooks_worker.py b/netbox/extras/webhooks_worker.py index 438231b7e11..4d6d8135e2c 100644 --- a/netbox/extras/webhooks_worker.py +++ b/netbox/extras/webhooks_worker.py @@ -5,36 +5,18 @@ from django_rq import job from jinja2.exceptions import TemplateError -from .conditions import ConditionSet from .constants import WEBHOOK_EVENT_TYPES from .webhooks import generate_signature logger = logging.getLogger('netbox.webhooks_worker') -def eval_conditions(webhook, data): - """ - Test whether the given data meets the conditions of the webhook (if any). Return True - if met or no conditions are specified. - """ - if not webhook.conditions: - return True - - logger.debug(f'Evaluating webhook conditions: {webhook.conditions}') - if ConditionSet(webhook.conditions).eval(data): - return True - - return False - - @job('default') -def process_webhook(webhook, model_name, event, data, timestamp, username, request_id=None, snapshots=None): +def process_webhook(event_rule, model_name, event, data, timestamp, username, request_id=None, snapshots=None): """ Make a POST request to the defined Webhook """ - # Evaluate webhook conditions (if any) - if not eval_conditions(webhook, data): - return + webhook = event_rule.action_object # Prepare context data for headers & body templates context = { diff --git a/netbox/netbox/context.py b/netbox/netbox/context.py index 5461a4e9479..56e41cb6311 100644 --- a/netbox/netbox/context.py +++ b/netbox/netbox/context.py @@ -2,9 +2,9 @@ __all__ = ( 'current_request', - 'webhooks_queue', + 'events_queue', ) current_request = ContextVar('current_request', default=None) -webhooks_queue = ContextVar('webhooks_queue', default=[]) +events_queue = ContextVar('events_queue', default=[]) diff --git a/netbox/netbox/middleware.py b/netbox/netbox/middleware.py index 18f350fd7b4..cb7d2c8bae2 100644 --- a/netbox/netbox/middleware.py +++ b/netbox/netbox/middleware.py @@ -10,7 +10,7 @@ from django.db.utils import InternalError from django.http import Http404, HttpResponseRedirect -from extras.context_managers import change_logging +from extras.context_managers import event_tracking from netbox.config import clear_config, get_config from netbox.views import handler_500 from utilities.api import is_api_request, rest_api_server_error @@ -42,8 +42,8 @@ def __call__(self, request): login_url = f'{settings.LOGIN_URL}?next={parse.quote(request.get_full_path_info())}' return HttpResponseRedirect(login_url) - # Enable the change_logging context manager and process the request. - with change_logging(request): + # Enable the event_tracking context manager and process the request. + with event_tracking(request): response = self.get_response(request) # Attach the unique request ID as an HTTP header. diff --git a/netbox/netbox/models/__init__.py b/netbox/netbox/models/__init__.py index 9d769669684..2c262b25867 100644 --- a/netbox/netbox/models/__init__.py +++ b/netbox/netbox/models/__init__.py @@ -30,7 +30,7 @@ class NetBoxFeatureSet( ExportTemplatesMixin, JournalingMixin, TagsMixin, - WebhooksMixin + EventRulesMixin ): class Meta: abstract = True @@ -44,7 +44,7 @@ def docs_url(self): # Base model classes # -class ChangeLoggedModel(ChangeLoggingMixin, CustomValidationMixin, WebhooksMixin, models.Model): +class ChangeLoggedModel(ChangeLoggingMixin, CustomValidationMixin, EventRulesMixin, models.Model): """ Base model for ancillary models; provides limited functionality for models which don't support NetBox's full feature set. diff --git a/netbox/netbox/models/features.py b/netbox/netbox/models/features.py index f39f3562081..ac9893e206c 100644 --- a/netbox/netbox/models/features.py +++ b/netbox/netbox/models/features.py @@ -35,7 +35,7 @@ 'JournalingMixin', 'SyncedDataMixin', 'TagsMixin', - 'WebhooksMixin', + 'EventRulesMixin', ) @@ -400,9 +400,9 @@ class Meta: abstract = True -class WebhooksMixin(models.Model): +class EventRulesMixin(models.Model): """ - Enables support for webhooks. + Enables support for event rules, which can be used to transmit webhooks or execute scripts automatically. """ class Meta: abstract = True @@ -555,7 +555,7 @@ def sync_data(self): 'journaling': JournalingMixin, 'synced_data': SyncedDataMixin, 'tags': TagsMixin, - 'webhooks': WebhooksMixin, + 'event_rules': EventRulesMixin, } registry['model_features'].update({ diff --git a/netbox/netbox/navigation/menu.py b/netbox/netbox/navigation/menu.py index 49aee354048..e01e65cc8c8 100644 --- a/netbox/netbox/navigation/menu.py +++ b/netbox/netbox/navigation/menu.py @@ -343,6 +343,7 @@ label=_('Integrations'), items=( get_model_item('core', 'datasource', _('Data Sources')), + get_model_item('extras', 'eventrule', _('Event Rules')), get_model_item('extras', 'webhook', _('Webhooks')), ), ), diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index 4a97711ff48..1181229f2f3 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -115,6 +115,9 @@ DEVELOPER = getattr(configuration, 'DEVELOPER', False) DOCS_ROOT = getattr(configuration, 'DOCS_ROOT', os.path.join(os.path.dirname(BASE_DIR), 'docs')) EMAIL = getattr(configuration, 'EMAIL', {}) +EVENTS_PIPELINE = getattr(configuration, 'EVENTS_PIPELINE', ( + 'extras.events.process_event_queue', +)) EXEMPT_VIEW_PERMISSIONS = getattr(configuration, 'EXEMPT_VIEW_PERMISSIONS', []) FIELD_CHOICES = getattr(configuration, 'FIELD_CHOICES', {}) FILE_UPLOAD_MAX_MEMORY_SIZE = getattr(configuration, 'FILE_UPLOAD_MAX_MEMORY_SIZE', 2621440) @@ -672,7 +675,7 @@ def _setting(name, default=None): # -# Django RQ (Webhooks backend) +# Django RQ (events backend) # if TASKS_REDIS_USING_SENTINEL: diff --git a/netbox/netbox/views/generic/bulk_views.py b/netbox/netbox/views/generic/bulk_views.py index 72d327453f2..4764642b3de 100644 --- a/netbox/netbox/views/generic/bulk_views.py +++ b/netbox/netbox/views/generic/bulk_views.py @@ -17,7 +17,7 @@ from django_tables2.export import TableExport from extras.models import ExportTemplate -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields @@ -279,7 +279,7 @@ def post(self, request): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -474,12 +474,12 @@ def post(self, request): return redirect(results_url) except (AbortTransaction, ValidationError): - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -632,12 +632,12 @@ def post(self, request, **kwargs): except ValidationError as e: messages.error(self.request, ", ".join(e.messages)) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -733,7 +733,7 @@ def post(self, request): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: form = self.form(initial={'pk': request.POST.getlist('pk')}) @@ -927,12 +927,12 @@ def post(self, request): raise PermissionsViolation except IntegrityError: - clear_webhooks.send(sender=self) + clear_events.send(sender=self) except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) if not form.errors: msg = "Added {} {} to {} {}.".format( diff --git a/netbox/netbox/views/generic/object_views.py b/netbox/netbox/views/generic/object_views.py index 99508c9e320..456c2e14f1d 100644 --- a/netbox/netbox/views/generic/object_views.py +++ b/netbox/netbox/views/generic/object_views.py @@ -11,7 +11,7 @@ from django.utils.html import escape from django.utils.safestring import mark_safe -from extras.signals import clear_webhooks +from extras.signals import clear_events from utilities.error_handlers import handle_protectederror from utilities.exceptions import AbortRequest, PermissionsViolation from utilities.forms import ConfirmationForm, restrict_form_fields @@ -300,7 +300,7 @@ def post(self, request, *args, **kwargs): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) else: logger.debug("Form validation failed") @@ -528,7 +528,7 @@ def post(self, request): except (AbortRequest, PermissionsViolation) as e: logger.debug(e.message) form.add_error(None, e.message) - clear_webhooks.send(sender=self) + clear_events.send(sender=self) return render(request, self.template_name, { 'object': instance, diff --git a/netbox/templates/extras/eventrule.html b/netbox/templates/extras/eventrule.html new file mode 100644 index 00000000000..86c330121cd --- /dev/null +++ b/netbox/templates/extras/eventrule.html @@ -0,0 +1,98 @@ +{% extends 'generic/object.html' %} +{% load helpers %} +{% load plugins %} +{% load i18n %} + +{% block content %} +
+
+
+
+ {% trans "Event Rule" %} +
+
+ + + + + + + + + + + + + +
{% trans "Name" %}{{ object.name }}
{% trans "Enabled" %}{% checkmark object.enabled %}
{% trans "Description" %}{{ object.description|placeholder }}
+
+
+
+
+ {% trans "Events" %} +
+
+ + + + + + + + + + + + + + + + + + + + + +
{% trans "Create" %}{% checkmark object.type_create %}
{% trans "Update" %}{% checkmark object.type_update %}
{% trans "Delete" %}{% checkmark object.type_delete %}
{% trans "Job start" %}{% checkmark object.type_job_start %}
{% trans "Job end" %}{% checkmark object.type_job_end %}
+
+
+ {% plugin_left_page object %} +
+
+
+
+ {% trans "Object Types" %} +
+
+ + {% for ct in object.content_types.all %} + + + + {% endfor %} +
{{ ct }}
+
+
+
+
+ {% trans "Conditions" %} +
+
+ {% if object.conditions %} + 
{{ object.conditions|json }}
+ {% else %} +

{% trans "None" %}

+ {% endif %} +
+
+ {% include 'inc/panels/custom_fields.html' %} + {% include 'inc/panels/tags.html' %} + {% plugin_right_page object %} +
+
+
+
+ {% plugin_full_width_page object %} +
+
+{% endblock %} diff --git a/netbox/templates/extras/webhook.html b/netbox/templates/extras/webhook.html index 5137b0103a7..c4b41faa144 100644 --- a/netbox/templates/extras/webhook.html +++ b/netbox/templates/extras/webhook.html @@ -16,39 +16,6 @@
{% trans "Name" %} {{ object.name }} - - {% trans "Enabled" %} - {% checkmark object.enabled %} - - - - -
-
- {% trans "Events" %} -
-
- - - - - - - - - - - - - - - - - - - - -
{% trans "Create" %}{% checkmark object.type_create %}
{% trans "Update" %}{% checkmark object.type_update %}
{% trans "Delete" %}{% checkmark object.type_delete %}
{% trans "Job start" %}{% checkmark object.type_job_start %}
{% trans "Job end" %}{% checkmark object.type_job_end %}
@@ -97,32 +64,6 @@
{% plugin_left_page object %}
-
-
- {% trans "Assigned Models" %} -
-
- - {% for ct in object.content_types.all %} - - - - {% endfor %} -
{{ ct }}
-
-
-
-
- {% trans "Conditions" %} -
-
- {% if object.conditions %} - 
{{ object.conditions|json }}
- {% else %} -

{% trans "None" %}

- {% endif %} -
-
{% trans "Additional Headers" %} diff --git a/netbox/utilities/forms/fields/fields.py b/netbox/utilities/forms/fields/fields.py index db5e4a30de0..d4d4ae19b8c 100644 --- a/netbox/utilities/forms/fields/fields.py +++ b/netbox/utilities/forms/fields/fields.py @@ -103,7 +103,7 @@ def __init__(self, *args, **kwargs): def prepare_value(self, value): if isinstance(value, InvalidJSONInput): return value - if value is None: + if value in ('', None): return '' return json.dumps(value, sort_keys=True, indent=4) diff --git a/netbox/utilities/forms/utils.py b/netbox/utilities/forms/utils.py index 64864a6c130..de8e2272770 100644 --- a/netbox/utilities/forms/utils.py +++ b/netbox/utilities/forms/utils.py @@ -128,10 +128,9 @@ def get_field_value(form, field_name): """ field = form.fields[field_name] - if form.is_bound: - if data := form.data.get(field_name): - if field.valid_value(data): - return data + if form.is_bound and (data := form.data.get(field_name)): + if hasattr(field, 'valid_value') and field.valid_value(data): + return data return form.get_initial_for_field(field, field_name)