diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index d6e960d5aa..fea2d28653 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -1640,7 +1640,7 @@ def record_custom_event(self, event_type, params): if not settings.custom_insights_events.enabled: return - event = create_custom_event(event_type, params) + event = create_custom_event(event_type, params, settings=settings) if event: self._custom_events.add(event, priority=self.priority) @@ -1653,7 +1653,7 @@ def record_ml_event(self, event_type, params): if not settings.ml_insights_events.enabled: return - event = create_custom_event(event_type, params, is_ml_event=True) + event = create_custom_event(event_type, params, settings=settings, is_ml_event=True) if event: self._ml_events.add(event, priority=self.priority) diff --git a/newrelic/common/package_version_utils.py b/newrelic/common/package_version_utils.py index 68320b897f..edefc4c0aa 100644 --- a/newrelic/common/package_version_utils.py +++ b/newrelic/common/package_version_utils.py @@ -13,6 +13,7 @@ # limitations under the License. import sys +import warnings try: from functools import cache as _cache_package_versions @@ -110,6 +111,23 @@ def _get_package_version(name): module = sys.modules.get(name, None) version = None + with warnings.catch_warnings(record=True): + for attr in VERSION_ATTRS: + try: + version = getattr(module, attr, None) + + # In certain cases like importlib_metadata.version, version is a callable + # function. + if callable(version): + continue + + # Cast any version specified as a list into a tuple. + version = tuple(version) if isinstance(version, list) else version + if version not in NULL_VERSIONS: + return version + except Exception: + pass + # importlib was introduced into the standard library starting in Python3.8. if "importlib" in sys.modules and hasattr(sys.modules["importlib"], "metadata"): try: @@ -126,20 +144,6 @@ def _get_package_version(name): except Exception: pass - for attr in VERSION_ATTRS: - try: - version = getattr(module, attr, None) - # In certain cases like importlib_metadata.version, version is a callable - # function. - if callable(version): - continue - # Cast any version specified as a list into a tuple. - version = tuple(version) if isinstance(version, list) else version - if version not in NULL_VERSIONS: - return version - except Exception: - pass - if "pkg_resources" in sys.modules: try: version = sys.modules["pkg_resources"].get_distribution(name).version diff --git a/newrelic/config.py b/newrelic/config.py index 6fe19705f2..1725c4eedb 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -45,6 +45,7 @@ from newrelic.common.log_file import initialize_logging from newrelic.common.object_names import expand_builtin_exception_name from newrelic.core import trace_cache +from newrelic.core.attribute import MAX_ATTRIBUTE_LENGTH from newrelic.core.config import ( Settings, apply_config_setting, @@ -443,6 +444,7 @@ def _process_configuration(section): ) _process_setting(section, "custom_insights_events.enabled", "getboolean", None) _process_setting(section, "custom_insights_events.max_samples_stored", "getint", None) + _process_setting(section, "custom_insights_events.max_attribute_value", "getint", MAX_ATTRIBUTE_LENGTH) _process_setting(section, "ml_insights_events.enabled", "getboolean", None) _process_setting(section, "distributed_tracing.enabled", "getboolean", None) _process_setting(section, "distributed_tracing.exclude_newrelic_header", "getboolean", None) diff --git a/newrelic/core/application.py b/newrelic/core/application.py index c681bc3f01..e1ada60aac 100644 --- a/newrelic/core/application.py +++ b/newrelic/core/application.py @@ -916,7 +916,7 @@ def record_custom_event(self, event_type, params): if settings is None or not settings.custom_insights_events.enabled: return - event = create_custom_event(event_type, params) + event = create_custom_event(event_type, params, settings=settings) if event: with self._stats_custom_lock: @@ -932,7 +932,7 @@ def record_ml_event(self, event_type, params): if settings is None or not settings.ml_insights_events.enabled: return - event = create_custom_event(event_type, params, is_ml_event=True) + event = create_custom_event(event_type, params, settings=settings, is_ml_event=True) if event: with self._stats_custom_lock: @@ -1506,7 +1506,9 @@ def harvest(self, shutdown=False, flexible=False): # Send metrics self._active_session.send_metric_data(self._period_start, period_end, metric_data) if dimensional_metric_data: - self._active_session.send_dimensional_metric_data(self._period_start, period_end, dimensional_metric_data) + self._active_session.send_dimensional_metric_data( + self._period_start, period_end, dimensional_metric_data + ) _logger.debug("Done sending data for harvest of %r.", self._app_name) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 483e23df80..27eb085b13 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -31,6 +31,7 @@ import newrelic.packages.six as six from newrelic.common.object_names import parse_exc_info +from newrelic.core.attribute import MAX_ATTRIBUTE_LENGTH from newrelic.core.attribute_filter import AttributeFilter try: @@ -717,6 +718,7 @@ def default_otlp_host(host): _settings.transaction_events.attributes.include = [] _settings.custom_insights_events.enabled = True +_settings.custom_insights_events.max_attribute_value = MAX_ATTRIBUTE_LENGTH _settings.ml_insights_events.enabled = False _settings.distributed_tracing.enabled = _environ_as_bool("NEW_RELIC_DISTRIBUTED_TRACING_ENABLED", default=True) @@ -810,6 +812,10 @@ def default_otlp_host(host): "NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_SAMPLES_STORED", CUSTOM_EVENT_RESERVOIR_SIZE ) +_settings.custom_insights_events.max_attribute_value = _environ_as_int( + "NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_ATTRIBUTE_VALUE", MAX_ATTRIBUTE_LENGTH +) + _settings.event_harvest_config.harvest_limits.ml_event_data = _environ_as_int( "NEW_RELIC_ML_INSIGHTS_EVENTS_MAX_SAMPLES_STORED", ML_EVENT_RESERVOIR_SIZE ) @@ -898,6 +904,7 @@ def default_otlp_host(host): _settings.machine_learning.inference_events_value.enabled = _environ_as_bool( "NEW_RELIC_MACHINE_LEARNING_INFERENCE_EVENT_VALUE_ENABLED", default=False ) +_settings.ml_insights_events.enabled = _environ_as_bool("NEW_RELIC_ML_INSIGHTS_EVENTS_ENABLED", default=False) def global_settings(): @@ -1170,6 +1177,14 @@ def apply_server_side_settings(server_side_config=None, settings=_settings): settings_snapshot.event_harvest_config.harvest_limits.ml_event_data / 12, ) + # Since the server does not override this setting we must override it here manually + # by caping it at the max value of 4095. + apply_config_setting( + settings_snapshot, + "custom_insights_events.max_attribute_value", + min(settings_snapshot.custom_insights_events.max_attribute_value, 4095), + ) + # This will be removed at some future point # Special case for account_id which will be sent instead of # cross_process_id in the future diff --git a/newrelic/core/custom_event.py b/newrelic/core/custom_event.py index b86dc25998..15becf437c 100644 --- a/newrelic/core/custom_event.py +++ b/newrelic/core/custom_event.py @@ -11,27 +11,37 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging import re import time -from newrelic.core.attribute import (check_name_is_string, check_name_length, - process_user_attribute, NameIsNotStringException, NameTooLongException, - MAX_NUM_USER_ATTRIBUTES, MAX_ML_ATTRIBUTE_LENGTH, MAX_NUM_ML_USER_ATTRIBUTES, MAX_ATTRIBUTE_LENGTH) - +from newrelic.core.attribute import ( + MAX_ML_ATTRIBUTE_LENGTH, + MAX_NUM_ML_USER_ATTRIBUTES, + MAX_NUM_USER_ATTRIBUTES, + NameIsNotStringException, + NameTooLongException, + check_name_is_string, + check_name_length, + process_user_attribute, +) +from newrelic.core.config import global_settings _logger = logging.getLogger(__name__) -EVENT_TYPE_VALID_CHARS_REGEX = re.compile(r'^[a-zA-Z0-9:_ ]+$') +EVENT_TYPE_VALID_CHARS_REGEX = re.compile(r"^[a-zA-Z0-9:_ ]+$") + + +class NameInvalidCharactersException(Exception): + pass -class NameInvalidCharactersException(Exception): pass def check_event_type_valid_chars(name): regex = EVENT_TYPE_VALID_CHARS_REGEX if not regex.match(name): raise NameInvalidCharactersException() + def process_event_type(name): """Perform all necessary validation on a potential event type. @@ -55,25 +65,22 @@ def process_event_type(name): check_event_type_valid_chars(name) except NameIsNotStringException: - _logger.debug('Event type must be a string. Dropping ' - 'event: %r', name) + _logger.debug("Event type must be a string. Dropping event: %r", name) return FAILED_RESULT except NameTooLongException: - _logger.debug('Event type exceeds maximum length. Dropping ' - 'event: %r', name) + _logger.debug("Event type exceeds maximum length. Dropping event: %r", name) return FAILED_RESULT except NameInvalidCharactersException: - _logger.debug('Event type has invalid characters. Dropping ' - 'event: %r', name) + _logger.debug("Event type has invalid characters. Dropping event: %r", name) return FAILED_RESULT else: return name -def create_custom_event(event_type, params, is_ml_event=False): +def create_custom_event(event_type, params, settings=None, is_ml_event=False): """Creates a valid custom event. Ensures that the custom event has a valid name, and also checks @@ -84,6 +91,7 @@ def create_custom_event(event_type, params, is_ml_event=False): Args: event_type (str): The type (name) of the custom event. params (dict): Attributes to add to the event. + settings: Optional config settings. is_ml_event (bool): Boolean indicating whether create_custom_event was called from record_ml_event for truncation purposes @@ -92,6 +100,7 @@ def create_custom_event(event_type, params, is_ml_event=False): None, if not successful. """ + settings = settings or global_settings() name = process_event_type(event_type) @@ -106,25 +115,30 @@ def create_custom_event(event_type, params, is_ml_event=False): max_length = MAX_ML_ATTRIBUTE_LENGTH max_num_attrs = MAX_NUM_ML_USER_ATTRIBUTES else: - max_length = MAX_ATTRIBUTE_LENGTH + max_length = settings.custom_insights_events.max_attribute_value max_num_attrs = MAX_NUM_USER_ATTRIBUTES key, value = process_user_attribute(k, v, max_length=max_length) if key: if len(attributes) >= max_num_attrs: - _logger.debug('Maximum number of attributes already ' - 'added to event %r. Dropping attribute: %r=%r', - name, key, value) + _logger.debug( + "Maximum number of attributes already added to event %r. Dropping attribute: %r=%r", + name, + key, + value, + ) else: attributes[key] = value except Exception: - _logger.debug('Attributes failed to validate for unknown reason. ' - 'Check traceback for clues. Dropping event: %r.', name, - exc_info=True) + _logger.debug( + "Attributes failed to validate for unknown reason. Check traceback for clues. Dropping event: %r.", + name, + exc_info=True, + ) return None intrinsics = { - 'type': name, - 'timestamp': int(1000.0 * time.time()), + "type": name, + "timestamp": int(1000.0 * time.time()), } event = [intrinsics, attributes] diff --git a/tests/agent_features/test_custom_events.py b/tests/agent_features/test_custom_events.py index d03feea291..0fb9c80bc2 100644 --- a/tests/agent_features/test_custom_events.py +++ b/tests/agent_features/test_custom_events.py @@ -14,128 +14,183 @@ import time +from testing_support.fixtures import ( + function_not_called, + override_application_settings, + reset_core_stats_engine, + validate_custom_event_count, + validate_custom_event_in_application_stats_engine, +) + from newrelic.api.application import application_instance as application from newrelic.api.background_task import background_task from newrelic.api.transaction import record_custom_event from newrelic.core.custom_event import process_event_type -from testing_support.fixtures import (reset_core_stats_engine, - validate_custom_event_count, - validate_custom_event_in_application_stats_engine, - override_application_settings, function_not_called) - # Test process_event_type() + def test_process_event_type_name_is_string(): - name = 'string' + name = "string" assert process_event_type(name) == name + def test_process_event_type_name_is_not_string(): name = 42 assert process_event_type(name) is None + def test_process_event_type_name_ok_length(): - ok_name = 'CustomEventType' + ok_name = "CustomEventType" assert process_event_type(ok_name) == ok_name + def test_process_event_type_name_too_long(): - too_long = 'a' * 256 + too_long = "a" * 256 assert process_event_type(too_long) is None + def test_process_event_type_name_valid_chars(): - valid_name = 'az09: ' + valid_name = "az09: " assert process_event_type(valid_name) == valid_name + def test_process_event_type_name_invalid_chars(): - invalid_name = '&' + invalid_name = "&" assert process_event_type(invalid_name) is None + _now = time.time() _intrinsics = { - 'type': 'FooEvent', - 'timestamp': _now, + "type": "FooEvent", + "timestamp": _now, } -_user_params = {'foo': 'bar'} +_user_params = {"foo": "bar"} _event = [_intrinsics, _user_params] + @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event) @background_task() def test_add_custom_event_to_transaction_stats_engine(): - record_custom_event('FooEvent', _user_params) + record_custom_event("FooEvent", _user_params) + @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event) def test_add_custom_event_to_application_stats_engine(): app = application() - record_custom_event('FooEvent', _user_params, application=app) + record_custom_event("FooEvent", _user_params, application=app) + @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_inside_transaction_bad_event_type(): - record_custom_event('!@#$%^&*()', {'foo': 'bar'}) + record_custom_event("!@#$%^&*()", {"foo": "bar"}) + @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_outside_transaction_bad_event_type(): app = application() - record_custom_event('!@#$%^&*()', {'foo': 'bar'}, application=app) + record_custom_event("!@#$%^&*()", {"foo": "bar"}, application=app) + + +_mixed_params = {"foo": "bar", 123: "bad key"} -_mixed_params = {'foo': 'bar', 123: 'bad key'} @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event) @background_task() def test_custom_event_inside_transaction_mixed_params(): - record_custom_event('FooEvent', _mixed_params) + record_custom_event("FooEvent", _mixed_params) + + +@override_application_settings({"custom_insights_events.max_attribute_value": 4095}) +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine([_intrinsics, {"foo": "bar", "bar": "a" * 4095}]) +@background_task() +def test_custom_event_inside_transaction_max_attribute_value(): + record_custom_event("FooEvent", {"foo": "bar", 123: "bad key", "bar": "a" * 5000}) + + +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine([_intrinsics, {"foo": "bar", "bar": "a" * 255}]) +@background_task() +def test_custom_event_inside_transaction_default_attribute_value(): + record_custom_event("FooEvent", {"foo": "bar", 123: "bad key", "bar": "a" * 5000}) + + +@override_application_settings({"custom_insights_events.max_attribute_value": 4095}) +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine([_intrinsics, {"foo": "bar", "bar": "a" * 4095}]) +def test_custom_event_outside_transaction_max_attribute_value(): + app = application() + record_custom_event("FooEvent", {"foo": "bar", 123: "bad key", "bar": "a" * 5000}, application=app) + + +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine([_intrinsics, {"foo": "bar", "bar": "a" * 255}]) +def test_custom_event_outside_transaction_default_attribute_value(): + app = application() + record_custom_event("FooEvent", {"foo": "bar", 123: "bad key", "bar": "a" * 5000}, application=app) + @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event) @background_task() def test_custom_event_outside_transaction_mixed_params(): app = application() - record_custom_event('FooEvent', _mixed_params, application=app) + record_custom_event("FooEvent", _mixed_params, application=app) + + +_bad_params = {"*" * 256: "too long", 123: "bad key"} +_event_with_no_params = [{"type": "FooEvent", "timestamp": _now}, {}] -_bad_params = {'*' * 256: 'too long', 123: 'bad key'} -_event_with_no_params = [{'type': 'FooEvent', 'timestamp': _now}, {}] @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event_with_no_params) @background_task() def test_custom_event_inside_transaction_bad_params(): - record_custom_event('FooEvent', _bad_params) + record_custom_event("FooEvent", _bad_params) + @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event_with_no_params) @background_task() def test_custom_event_outside_transaction_bad_params(): app = application() - record_custom_event('FooEvent', _bad_params, application=app) + record_custom_event("FooEvent", _bad_params, application=app) + @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_params_not_a_dict(): - record_custom_event('ParamsListEvent', ['not', 'a', 'dict']) + record_custom_event("ParamsListEvent", ["not", "a", "dict"]) + # Tests for Custom Events configuration settings -@override_application_settings({'collect_custom_events': False}) + +@override_application_settings({"collect_custom_events": False}) @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_settings_check_collector_flag(): - record_custom_event('FooEvent', _user_params) + record_custom_event("FooEvent", _user_params) + -@override_application_settings({'custom_insights_events.enabled': False}) +@override_application_settings({"custom_insights_events.enabled": False}) @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_settings_check_custom_insights_enabled(): - record_custom_event('FooEvent', _user_params) + record_custom_event("FooEvent", _user_params) + # Test that record_custom_event() methods will short-circuit. # @@ -143,15 +198,17 @@ def test_custom_event_settings_check_custom_insights_enabled(): # `create_custom_event()` function is not called, in order to avoid the # event_type and attribute processing. -@override_application_settings({'custom_insights_events.enabled': False}) -@function_not_called('newrelic.api.transaction', 'create_custom_event') + +@override_application_settings({"custom_insights_events.enabled": False}) +@function_not_called("newrelic.api.transaction", "create_custom_event") @background_task() def test_transaction_create_custom_event_not_called(): - record_custom_event('FooEvent', _user_params) + record_custom_event("FooEvent", _user_params) + -@override_application_settings({'custom_insights_events.enabled': False}) -@function_not_called('newrelic.core.application', 'create_custom_event') +@override_application_settings({"custom_insights_events.enabled": False}) +@function_not_called("newrelic.core.application", "create_custom_event") @background_task() def test_application_create_custom_event_not_called(): app = application() - record_custom_event('FooEvent', _user_params, application=app) + record_custom_event("FooEvent", _user_params, application=app) diff --git a/tests/agent_unittests/test_package_version_utils.py b/tests/agent_unittests/test_package_version_utils.py index 5ed689ea2a..b57c91aa60 100644 --- a/tests/agent_unittests/test_package_version_utils.py +++ b/tests/agent_unittests/test_package_version_utils.py @@ -13,8 +13,10 @@ # limitations under the License. import sys +import warnings import pytest +import six from testing_support.validators.validate_function_called import validate_function_called from newrelic.common.package_version_utils import ( @@ -66,30 +68,26 @@ def cleared_package_version_cache(): ("version_tuple", [3, 1, "0b2"], "3.1.0b2"), ), ) -def test_get_package_version(attr, value, expected_value): +def test_get_package_version(monkeypatch, attr, value, expected_value): # There is no file/module here, so we monkeypatch # pytest instead for our purposes - setattr(pytest, attr, value) + monkeypatch.setattr(pytest, attr, value, raising=False) version = get_package_version("pytest") assert version == expected_value - delattr(pytest, attr) # This test only works on Python 3.7 @SKIP_IF_IMPORTLIB_METADATA -def test_skips_version_callables(): +def test_skips_version_callables(monkeypatch): # There is no file/module here, so we monkeypatch # pytest instead for our purposes - setattr(pytest, "version", lambda x: "1.2.3.4") - setattr(pytest, "version_tuple", [3, 1, "0b2"]) + monkeypatch.setattr(pytest, "version", lambda x: "1.2.3.4", raising=False) + monkeypatch.setattr(pytest, "version_tuple", [3, 1, "0b2"], raising=False) version = get_package_version("pytest") assert version == "3.1.0b2" - delattr(pytest, "version") - delattr(pytest, "version_tuple") - # This test only works on Python 3.7 @SKIP_IF_IMPORTLIB_METADATA @@ -102,13 +100,12 @@ def test_skips_version_callables(): ("version_tuple", [3, 1, "0b2"], (3, 1, "0b2")), ), ) -def test_get_package_version_tuple(attr, value, expected_value): +def test_get_package_version_tuple(monkeypatch, attr, value, expected_value): # There is no file/module here, so we monkeypatch # pytest instead for our purposes - setattr(pytest, attr, value) + monkeypatch.setattr(pytest, attr, value, raising=False) version = get_package_version_tuple("pytest") assert version == expected_value - delattr(pytest, attr) @SKIP_IF_NOT_IMPORTLIB_METADATA @@ -132,10 +129,28 @@ def test_pkg_resources_metadata(): assert version not in NULL_VERSIONS, version +def _getattr_deprecation_warning(attr): + if attr == "__version__": + warnings.warn("Testing deprecation warnings.", DeprecationWarning) + return "3.2.1" + else: + raise NotImplementedError() + + +@pytest.mark.skipif(six.PY2, reason="Can't add Deprecation in __version__ in Python 2.") +def test_deprecation_warning_suppression(monkeypatch, recwarn): + # Add fake module to be deleted later + monkeypatch.setattr(pytest, "__getattr__", _getattr_deprecation_warning, raising=False) + + assert get_package_version("pytest") == "3.2.1" + + assert not recwarn.list, "Warnings not suppressed." + + def test_version_caching(monkeypatch): # Add fake module to be deleted later sys.modules["mymodule"] = sys.modules["pytest"] - setattr(pytest, "__version__", "1.0.0") + monkeypatch.setattr(pytest, "__version__", "1.0.0", raising=False) version = get_package_version("mymodule") assert version not in NULL_VERSIONS, version