feat(transport): Client Report Support (#1181)

This adds support for client reports to the python SDK. This will cause the SDK
to send a report once every 30 seconds or once a minute. After 30 seconds it
will attempt to attach the report to a scheduled envelope if there is one, after
60 seconds it will send it as a separate envelope. Attempts of sending are
only made as a byproduct of attempted event / envelope sending or an
explicit flush.

Author: mitsuhiko

.vscode/settings.json

@@ -1,3 +1,4 @@
 {
-    "python.pythonPath": ".venv/bin/python"
+    "python.pythonPath": ".venv/bin/python",
+    "python.formatting.provider": "black"
 }

scripts/init_serverless_sdk.py

@@ -51,16 +51,23 @@ def extract_and_load_lambda_function_module(self, module_path):
             # Supported python versions are 2.7, 3.6, 3.7, 3.8
             if py_version >= (3, 5):
                 import importlib.util
-                spec = importlib.util.spec_from_file_location(module_name, module_file_path)
+
+                spec = importlib.util.spec_from_file_location(
+                    module_name, module_file_path
+                )
                 self.lambda_function_module = importlib.util.module_from_spec(spec)
                 spec.loader.exec_module(self.lambda_function_module)
             elif py_version[0] < 3:
                 import imp
-                self.lambda_function_module = imp.load_source(module_name, module_file_path)
+
+                self.lambda_function_module = imp.load_source(
+                    module_name, module_file_path
+                )
             else:
                 raise ValueError("Python version %s is not supported." % py_version)
         else:
             import importlib
+
             self.lambda_function_module = importlib.import_module(module_path)
 
     def get_lambda_handler(self):

sentry_sdk/_types.py

@@ -37,7 +37,14 @@
     NotImplementedType = Any
 
     EventDataCategory = Literal[
-        "default", "error", "crash", "transaction", "security", "attachment", "session"
+        "default",
+        "error",
+        "crash",
+        "transaction",
+        "security",
+        "attachment",
+        "session",
+        "internal",
     ]
     SessionStatus = Literal["ok", "exited", "crashed", "abnormal"]
     EndpointType = Literal["store", "envelope"]

sentry_sdk/client.py

@@ -243,6 +243,9 @@ def _should_capture(
             self.options["sample_rate"] < 1.0
             and random.random() >= self.options["sample_rate"]
         ):
+            # record a lost event if we did not sample this.
+            if self.transport:
+                self.transport.record_lost_event("sample_rate", data_category="error")
             return False
 
         if self._is_ignored_error(event, hint):

sentry_sdk/consts.py

@@ -75,6 +75,7 @@ def __init__(
         traces_sampler=None,  # type: Optional[TracesSampler]
         auto_enabling_integrations=True,  # type: bool
         auto_session_tracking=True,  # type: bool
+        send_client_reports=True,  # type: bool
         _experiments={},  # type: Experiments  # noqa: B006
     ):
         # type: (...) -> None

sentry_sdk/envelope.py

@@ -2,7 +2,7 @@
 import json
 import mimetypes
 
-from sentry_sdk._compat import text_type
+from sentry_sdk._compat import text_type, PY2
 from sentry_sdk._types import MYPY
 from sentry_sdk.session import Session
 from sentry_sdk.utils import json_dumps, capture_internal_exceptions
@@ -18,6 +18,14 @@
     from sentry_sdk._types import Event, EventDataCategory
 
 
+def parse_json(data):
+    # type: (Union[bytes, text_type]) -> Any
+    # on some python 3 versions this needs to be bytes
+    if not PY2 and isinstance(data, bytes):
+        data = data.decode("utf-8", "replace")
+    return json.loads(data)
+
+
 class Envelope(object):
     def __init__(
         self,
@@ -114,7 +122,7 @@ def deserialize_from(
         cls, f  # type: Any
     ):
         # type: (...) -> Envelope
-        headers = json.loads(f.readline())
+        headers = parse_json(f.readline())
         items = []
         while 1:
             item = Item.deserialize_from(f)
@@ -236,6 +244,8 @@ def data_category(self):
             return "transaction"
         elif ty == "event":
             return "error"
+        elif ty == "client_report":
+            return "internal"
         else:
             return "default"
 
@@ -284,11 +294,11 @@ def deserialize_from(
         line = f.readline().rstrip()
         if not line:
             return None
-        headers = json.loads(line)
+        headers = parse_json(line)
         length = headers["length"]
         payload = f.read(length)
         if headers.get("type") in ("event", "transaction"):
-            rv = cls(headers=headers, payload=PayloadRef(json=json.loads(payload)))
+            rv = cls(headers=headers, payload=PayloadRef(json=parse_json(payload)))
         else:
             rv = cls(headers=headers, payload=payload)
         f.readline()

sentry_sdk/tracing.py

@@ -507,13 +507,22 @@ def finish(self, hub=None):
             # This transaction is already finished, ignore.
             return None
 
+        hub = hub or self.hub or sentry_sdk.Hub.current
+        client = hub.client
+
         # This is a de facto proxy for checking if sampled = False
         if self._span_recorder is None:
             logger.debug("Discarding transaction because sampled = False")
-            return None
 
-        hub = hub or self.hub or sentry_sdk.Hub.current
-        client = hub.client
+            # This is not entirely accurate because discards here are not
+            # exclusively based on sample rate but also traces sampler, but
+            # we handle this the same here.
+            if client and client.transport:
+                client.transport.record_lost_event(
+                    "sample_rate", data_category="transaction"
+                )
+
+            return None
 
         if client is None:
             # We have no client and therefore nowhere to send this transaction.