4559 config context rendering

netbox/dcim/api/views.py

@@ -24,7 +24,7 @@
     VirtualChassis,
 )
 from extras.api.serializers import RenderedGraphSerializer
-from extras.api.views import CustomFieldModelViewSet
+from extras.api.views import ConfigContextQuerySetMixin, CustomFieldModelViewSet
 from extras.models import Graph
 from ipam.models import Prefix, VLAN
 from utilities.api import (
@@ -336,7 +336,7 @@ class PlatformViewSet(ModelViewSet):
 # Devices
 #
 
-class DeviceViewSet(CustomFieldModelViewSet):
+class DeviceViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin):
     queryset = Device.objects.prefetch_related(
         'device_type__manufacturer', 'device_role', 'tenant', 'platform', 'site', 'rack', 'parent_bay',
         'virtual_chassis__master', 'primary_ip4__nat_outside', 'primary_ip6__nat_outside', 'tags',

netbox/dcim/models/devices.py

@@ -15,6 +15,7 @@
 from dcim.choices import *
 from dcim.constants import *
 from extras.models import ChangeLoggedModel, ConfigContextModel, CustomFieldModel, TaggedItem
+from extras.querysets import ConfigContextModelQuerySet
 from extras.utils import extras_features
 from utilities.choices import ColorChoices
 from utilities.fields import ColorField, NaturalOrderingField
@@ -594,7 +595,7 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
     )
     tags = TaggableManager(through=TaggedItem)
 
-    objects = RestrictedQuerySet.as_manager()
+    objects = ConfigContextModelQuerySet.as_manager()
 
     csv_headers = [
         'name', 'device_role', 'tenant', 'manufacturer', 'device_type', 'platform', 'serial', 'asset_tag', 'status',

netbox/dcim/views.py

@@ -1163,7 +1163,7 @@ def get(self, request, pk):
 
 
 class DeviceConfigContextView(ObjectConfigContextView):
-    queryset = Device.objects.all()
+    queryset = Device.objects.annotate_config_context_data()
     base_template = 'dcim/device.html'
 
 

netbox/extras/api/views.py

@@ -26,6 +26,29 @@
 from . import serializers
 
 
+class ConfigContextQuerySetMixin:
+    """
+    Used by views that work with config context models (device and virtual machine).
+    Provides a get_queryset() method which deals with adding the config context
+    data annotation or not.
+    """
+
+    def get_queryset(self):
+        """
+        Build the proper queryset based on the request context
+
+        If the `brief` query param equates to True or the `exclude` query param
+        includes `config_context` as a value, return the base queryset.
+
+        Else, return the queryset annotated with config context data
+        """
+
+        request = self.get_serializer_context()['request']
+        if request.query_params.get('brief') or 'config_context' in request.query_params.get('exclude', []):
+            return self.queryset
+        return self.queryset.annotate_config_context_data()
+
+
 class ExtrasRootView(APIRootView):
     """
     Extras API root view

netbox/extras/models/models.py

@@ -542,8 +542,16 @@ def get_config_context(self):
 
         # Compile all config data, overwriting lower-weight values with higher-weight values where a collision occurs
         data = OrderedDict()
-        for context in ConfigContext.objects.get_for_object(self):
-            data = deepmerge(data, context.data)
+
+        if not hasattr(self, 'config_context_data'):
+            # The annotation is not available, so we fall back to manually querying for the config context objects
+            config_context_data = ConfigContext.objects.get_for_object(self, aggregate_data=True)
+        else:
+            # The attribute may exist, but the annotated value could be None if there is no config context data
+            config_context_data = self.config_context_data or []
+
+        for context in config_context_data:
+            data = deepmerge(data, context)
 
         # If the object has local config context data defined, merge it last
         if self.local_context_data:

netbox/extras/querysets.py

@@ -1,7 +1,8 @@
 from collections import OrderedDict
 
-from django.db.models import Q, QuerySet
+from django.db.models import OuterRef, Subquery, Q
 
+from utilities.query_functions import EmptyGroupByJSONBAgg, OrderableJSONBAgg
 from utilities.querysets import RestrictedQuerySet
 
 
@@ -23,9 +24,12 @@ def __iter__(self):
 
 class ConfigContextQuerySet(RestrictedQuerySet):
 
-    def get_for_object(self, obj):
+    def get_for_object(self, obj, aggregate_data=False):
         """
         Return all applicable ConfigContexts for a given object. Only active ConfigContexts will be included.
+
+        Args:
+          aggregate_data: If True, use the JSONBAgg aggregate function to return only the list of JSON data objects
         """
 
         # `device_role` for Device; `role` for VirtualMachine
@@ -45,7 +49,7 @@ def get_for_object(self, obj):
         else:
             regions = []
 
-        return self.filter(
+        queryset = self.filter(
             Q(regions__in=regions) | Q(regions=None),
             Q(sites=obj.site) | Q(sites=None),
             Q(roles=role) | Q(roles=None),
@@ -57,3 +61,72 @@ def get_for_object(self, obj):
             Q(tags__slug__in=obj.tags.slugs()) | Q(tags=None),
             is_active=True,
         ).order_by('weight', 'name')
+
+        if aggregate_data:
+            return queryset.aggregate(
+                config_context_data=OrderableJSONBAgg('data', ordering=['weight', 'name'])
+            )['config_context_data']
+
+        return queryset
+
+
+class ConfigContextModelQuerySet(RestrictedQuerySet):
+    """
+    QuerySet manager used by models which support ConfigContext (device and virtual machine).
+
+    Includes a method which appends an annotation of aggregated config context JSON data objects. This is
+    implemented as a subquery which performs all the joins necessary to filter relevant config context objects.
+    This offers a substantial performance gain over ConfigContextQuerySet.get_for_object() when dealing with
+    multiple objects.
+
+    This allows the annotation to be entirely optional.
+    """
+
+    def annotate_config_context_data(self):
+        """
+        Attach the subquery annotation to the base queryset
+        """
+        from extras.models import ConfigContext
+        return self.annotate(
+            config_context_data=Subquery(
+                ConfigContext.objects.filter(
+                    self._get_config_context_filters()
+                ).annotate(
+                    _data=EmptyGroupByJSONBAgg('data', ordering=['weight', 'name'])
+                ).values("_data")
+            )
+        )
+
+    def _get_config_context_filters(self):
+        # Construct the set of Q objects for the specific object types
+        base_query = Q(
+            Q(platforms=OuterRef('platform')) | Q(platforms=None),
+            Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None),
+            Q(tenants=OuterRef('tenant')) | Q(tenants=None),
+            Q(tags=OuterRef('tags')) | Q(tags=None),
+            is_active=True,
+        )
+
+        if self.model._meta.model_name == 'device':
+            base_query.add((Q(roles=OuterRef('device_role')) | Q(roles=None)), Q.AND)
+            base_query.add((Q(sites=OuterRef('site')) | Q(sites=None)), Q.AND)
+            region_field = 'site__region'
+
+        elif self.model._meta.model_name == 'virtualmachine':
+            base_query.add((Q(roles=OuterRef('role')) | Q(roles=None)), Q.AND)
+            base_query.add((Q(cluster_groups=OuterRef('cluster__group')) | Q(cluster_groups=None)), Q.AND)
+            base_query.add((Q(clusters=OuterRef('cluster')) | Q(clusters=None)), Q.AND)
+            base_query.add((Q(sites=OuterRef('cluster__site')) | Q(sites=None)), Q.AND)
+            region_field = 'cluster__site__region'
+
+        base_query.add(
+            (Q(
+                regions__tree_id=OuterRef(f'{region_field}__tree_id'),
+                regions__level__lte=OuterRef(f'{region_field}__level'),
+                regions__lft__lte=OuterRef(f'{region_field}__lft'),
+                regions__rght__gte=OuterRef(f'{region_field}__rght'),
+            ) | Q(regions=None)),
+            Q.AND
+        )
+
+        return base_query