From 76178c2412e32e73a18fcf3676b267775a5c68a5 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Fri, 12 Mar 2021 16:40:38 -0500 Subject: [PATCH 01/15] Initial mypy type hint for ddtrace core --- ddtrace/internal/_encoding.pyi | 8 ++++ ddtrace/internal/_rand.pyi | 2 + ddtrace/tracer.py | 69 +++++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 ddtrace/internal/_encoding.pyi create mode 100644 ddtrace/internal/_rand.pyi diff --git a/ddtrace/internal/_encoding.pyi b/ddtrace/internal/_encoding.pyi new file mode 100644 index 00000000000..7fdb7331b33 --- /dev/null +++ b/ddtrace/internal/_encoding.pyi @@ -0,0 +1,8 @@ +from typing import Any +from typing import Union + + +class MsgpackEncoder(object): + content_type: str + + def _decode(self, data: Union[str, bytes]) -> Any: ... \ No newline at end of file diff --git a/ddtrace/internal/_rand.pyi b/ddtrace/internal/_rand.pyi new file mode 100644 index 00000000000..6e44c87b566 --- /dev/null +++ b/ddtrace/internal/_rand.pyi @@ -0,0 +1,2 @@ +def seed() -> None: ... +def rand64bits(check_pid: bool = True) -> int: ... \ No newline at end of file diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index a30d91e3dda..df6df270fbf 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -4,6 +4,11 @@ from os import environ from os import getpid import sys +from typing import Dict +from typing import Callable +from typing import List +from typing import Optional +from typing import Union from . import _hooks from . import compat @@ -14,8 +19,10 @@ from .constants import VERSION_KEY from .context import Context from .ext import system +from .ext import SpanTypes from .ext.priority import AUTO_KEEP from .ext.priority import AUTO_REJECT +from .filters import TraceFilter from .internal import _rand from .internal import agent from .internal import debug @@ -31,7 +38,10 @@ from .sampler import DatadogSampler from .sampler import RateByServiceSampler from .sampler import RateSampler -from .settings import config + +from ddtrace import config + +# from .settings import config # TODO: type hint this bad boy OR just import directly from ddtrace from .span import Span from .utils.deprecation import deprecated from .utils.formats import asbool @@ -68,7 +78,11 @@ class Tracer(object): trace = tracer.trace('app.request', 'web-server').finish() """ - def __init__(self, url=None, dogstatsd_url=None): + def __init__( + self, + url=None, # type: Optional[str] + dogstatsd_url=None # type: Optional[str] + ): """ Create a new ``Tracer`` instance. A global tracer is already initialized for common usage, so there is no need to initialize your own ``Tracer``. @@ -80,7 +94,7 @@ def __init__(self, url=None, dogstatsd_url=None): self.sampler = None self.priority_sampler = None self._runtime_worker = None - self._filters = [] + self._filters = [] # type: List[TraceFilter] configure_kwargs = {} writer = None @@ -134,6 +148,7 @@ def __init__(self, url=None, dogstatsd_url=None): self._hooks = _hooks.Hooks() def on_start_span(self, func): + # type: (Callable) -> Callable """Register a function to execute when a span start. Can be used as a decorator. @@ -145,6 +160,7 @@ def on_start_span(self, func): return func def deregister_on_start_span(self, func): + # type: (Callable) -> Callable """Unregister a function registered to execute when a span starts. Can be used as a decorator. @@ -159,7 +175,7 @@ def deregister_on_start_span(self, func): def debug_logging(self): return self.log.isEnabledFor(logging.DEBUG) - @debug_logging.setter + @debug_logging.setter # type: ignore[misc] @deprecated(message="Use logging.setLevel instead", version="1.0.0") def debug_logging(self, value): self.log.setLevel(logging.DEBUG if value else logging.WARN) @@ -173,6 +189,7 @@ def global_excepthook(self, tp, value, traceback): """The global tracer except hook.""" def get_call_context(self, *args, **kwargs): + # type: (...) -> Context """ Return the current active ``Context`` for this traced execution. This method is automatically called in the ``tracer.trace()``, but it can be used in the application @@ -333,7 +350,14 @@ def configure( msg = "- DATADOG TRACER DIAGNOSTIC - %s" % agent_error self._log_compat(logging.WARNING, msg) - def start_span(self, name, child_of=None, service=None, resource=None, span_type=None): + def start_span( + self, + name, # type: str + child_of=None, # type: Optional[Union[Span, Context]] + service=None, # type: Optional[str] + resource=None, # type: Optional[str] + span_type=None # type: Optional[Union[str, SpanTypes]] + ): """ Return a span that will trace an operation called `name`. This method allows parenting using the ``child_of`` kwarg. If it's missing, the newly created span is a @@ -389,9 +413,9 @@ def start_span(self, name, child_of=None, service=None, resource=None, span_type if parent: service = parent.service else: - service = config.service + service = config.service # type: ignore[attr-defined] - mapped_service = config.service_mapping.get(service, service) + mapped_service = config.service_mapping.get(service, service) # type: ignore[attr-defined] if trace_id: # child_of a non-empty context, so either a local child span or from a remote context @@ -423,7 +447,7 @@ def start_span(self, name, child_of=None, service=None, resource=None, span_type ) span.metrics[system.PID] = self._pid or getpid() span.meta["runtime-id"] = get_runtime_id() - if config.report_hostname: + if config.report_hostname: # type: ignore[attr-defined] span.meta[HOSTNAME_KEY] = hostname.get_hostname() # add tags to root span to correlate trace with runtime metrics # only applied to spans with types that are internal to applications @@ -461,8 +485,8 @@ def start_span(self, name, child_of=None, service=None, resource=None, span_type if self.tags: span.set_tags(self.tags) - if config.env: - span._set_str_tag(ENV_KEY, config.env) + if config.env: # type: ignore[attr-defined] + span._set_str_tag(ENV_KEY, config.env) # type: ignore[attr-defined] # Only set the version tag on internal spans. if config.version: @@ -471,10 +495,10 @@ def start_span(self, name, child_of=None, service=None, resource=None, span_type # 2. the span is not the root, but the root span's service matches the span's service # and the root span has a version tag # then the span belongs to the user application and so set the version tag - if (root_span is None and service == config.service) or ( + if (root_span is None and service == config.service) or ( # type: ignore[attr-defined] root_span and root_span.service == service and VERSION_KEY in root_span.meta ): - span._set_str_tag(VERSION_KEY, config.version) + span._set_str_tag(VERSION_KEY, config.version) # type: ignore[attr-defined] # add it to the current context context.add_span(span) @@ -554,7 +578,13 @@ def _log_compat(self, level, msg): else: self.log.log(level, msg) - def trace(self, name, service=None, resource=None, span_type=None): + def trace( + self, + name, # type: str + service=None, # type: Optional[str] + resource=None, # type: Optional[str] + span_type=None # type: Optional[Union[str, SpanTypes]] + ): """ Return a span that will trace an operation called `name`. The context that created the span as well as the span parenting, are automatically handled by the tracing @@ -602,6 +632,7 @@ def trace(self, name, service=None, resource=None, span_type=None): ) def current_root_span(self): + # type: () -> Optional[Span] """Returns the root span of the current context. This is useful for attaching information related to the trace as a @@ -621,6 +652,7 @@ def current_root_span(self): return None def current_span(self): + # type: () -> Optional[Span] """ Return the active span for the current call context or ``None`` if no spans are available. @@ -631,6 +663,7 @@ def current_span(self): return None def write(self, spans): + # type: (List[Span]) -> None """ Send the trace to the writer to enqueue the spans list in the agent sending queue. @@ -660,7 +693,13 @@ def set_service_info(self, *args, **kwargs): """Set the information about the given service.""" return - def wrap(self, name=None, service=None, resource=None, span_type=None): + def wrap( + self, + name=None, # type: Optional[str] + service=None, # type: Optional[str] + resource=None, # type: Optional[str] + span_type=None # type: Optional[str] + ): """ A decorator used to trace an entire function. If the traced function is a coroutine, it traces the coroutine execution when is awaited. @@ -750,6 +789,7 @@ def func_wrapper(*args, **kwargs): return wrap_decorator def set_tags(self, tags): + # type: (Dict[str, str]) -> None """Set some tags at the tracer level. This will append those tags to each span created by the tracer. @@ -758,6 +798,7 @@ def set_tags(self, tags): self.tags.update(tags) def shutdown(self, timeout=None): + # type: (Optional[float]) -> None """Shutdown the tracer. This will stop the background writer/worker and flush any finished traces in the buffer. From bcb73a9bb14458cd9f62dcae7ce00cdcc056789a Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Fri, 12 Mar 2021 16:41:37 -0500 Subject: [PATCH 02/15] Initial mypy type checks for ddtrace core --- ddtrace/__init__.py | 2 +- ddtrace/_hooks.py | 4 +++- ddtrace/compat.py | 29 ++++++++++++++++------------- ddtrace/filters.py | 2 +- ddtrace/monkey.py | 4 ++-- ddtrace/provider.py | 2 +- ddtrace/sampler.py | 4 ++-- mypy.ini | 2 +- 8 files changed, 27 insertions(+), 22 deletions(-) diff --git a/ddtrace/__init__.py b/ddtrace/__init__.py index 8095ce3a5d6..b7959a0e77d 100644 --- a/ddtrace/__init__.py +++ b/ddtrace/__init__.py @@ -3,7 +3,7 @@ from .monkey import patch # noqa: E402 from .monkey import patch_all from .pin import Pin # noqa: E402 -from .settings import config # noqa: E402 +from .settings import config # noqa: E402 # Just move the init to here from .span import Span # noqa: E402 from .tracer import Tracer # noqa: E402 from .utils.deprecation import deprecated # noqa: E402 diff --git a/ddtrace/_hooks.py b/ddtrace/_hooks.py index c182e543866..a89497a1c10 100644 --- a/ddtrace/_hooks.py +++ b/ddtrace/_hooks.py @@ -1,5 +1,7 @@ import collections from copy import deepcopy +from typing import Dict +from typing import Set from .internal.logger import get_logger from .vendor import attr @@ -20,7 +22,7 @@ def on_request(span, request, response): pass """ - _hooks = attr.ib(init=False, factory=lambda: collections.defaultdict(set)) + _hooks = attr.ib(init=False, factory=lambda: collections.defaultdict(set)) # type: Dict[str, Set] def __deepcopy__(self, memodict=None): hooks = Hooks() diff --git a/ddtrace/compat.py b/ddtrace/compat.py index e69614c3e67..76fc8a6bc1e 100644 --- a/ddtrace/compat.py +++ b/ddtrace/compat.py @@ -29,17 +29,17 @@ PYTHON_INTERPRETER = platform.python_implementation() try: - StringIO = six.moves.cStringIO + StringIO = six.moves.cStringIO # type: ignore[attr-defined] except ImportError: StringIO = six.StringIO -httplib = six.moves.http_client -urlencode = six.moves.urllib.parse.urlencode -parse = six.moves.urllib.parse -Queue = six.moves.queue.Queue +httplib = six.moves.http_client # type: ignore[attr-defined] +urlencode = six.moves.urllib.parse.urlencode # type: ignore[attr-defined] +parse = six.moves.urllib.parse # type: ignore[attr-defined] +Queue = six.moves.queue.Queue # type: ignore[attr-defined] iteritems = six.iteritems reraise = six.reraise -reload_module = six.moves.reload_module +reload_module = six.moves.reload_module # type: ignore[attr-defined] stringify = six.text_type string_type = six.string_types[0] @@ -52,7 +52,7 @@ if PYTHON_VERSION_INFO >= (3, 7): pattern_type = re.Pattern else: - pattern_type = re._pattern_type + pattern_type = re._pattern_type # type: ignore[misc,attr-defined] def is_integer(obj): @@ -71,6 +71,7 @@ def is_integer(obj): from time import time as _time def time_ns(): + # type: () -> int return int(_time() * 10e5) * 1000 @@ -85,15 +86,17 @@ def time_ns(): except ImportError: def monotonic_ns(): + # type: () -> int return int(monotonic() * 1e9) try: from time import process_time_ns except ImportError: - from time import clock as _process_time + from time import clock as _process_time # type: ignore[attr-defined] def process_time_ns(): + # type: () -> int return int(_process_time() * 1e9) @@ -104,10 +107,10 @@ def process_time_ns(): if sys.version_info.major < 3: - if isinstance(threading.current_thread(), threading._MainThread): + if isinstance(threading.current_thread(), threading._MainThread): # type: ignore[attr-defined] main_thread = threading.current_thread() else: - main_thread = threading._shutdown.im_self + main_thread = threading._shutdown.im_self # type: ignore[attr-defined] else: main_thread = threading.main_thread() @@ -152,10 +155,10 @@ def func_wrapper(*args, **kwargs): # asyncio is missing so we can't have coroutines; these # functions are used only to ensure code executions in case # of an unexpected behavior - def iscoroutinefunction(fn): + def iscoroutinefunction(fn): # type: ignore return False - def make_async_decorator(tracer, fn, *params, **kw_params): + def make_async_decorator(tracer, fn, *params, **kw_params): # type: ignore return fn @@ -200,7 +203,7 @@ def get_connection_response(conn): try: import contextvars # noqa except ImportError: - from ddtrace.vendor import contextvars # noqa + from ddtrace.vendor import contextvars # type: ignore CONTEXTVARS_IS_AVAILABLE = False else: diff --git a/ddtrace/filters.py b/ddtrace/filters.py index 5ab2bc29a61..8d4e54f4c95 100644 --- a/ddtrace/filters.py +++ b/ddtrace/filters.py @@ -9,7 +9,7 @@ from .ext import http -class TraceFilter(six.with_metaclass(abc.ABCMeta)): +class TraceFilter(six.with_metaclass(abc.ABCMeta)): # type: ignore[misc] @abc.abstractmethod def process_trace(self, trace): # type: (List[Span]) -> Optional[List[Span]] diff --git a/ddtrace/monkey.py b/ddtrace/monkey.py index bb1071b2fc7..82318a1c139 100644 --- a/ddtrace/monkey.py +++ b/ddtrace/monkey.py @@ -14,7 +14,7 @@ from ddtrace.vendor.wrapt.importer import when_imported from .internal.logger import get_logger -from .settings import config +from .settings import config # type: ignore from .utils import formats @@ -64,7 +64,7 @@ "pylons": False, "pyramid": False, # Auto-enable logging if the environment variable DD_LOGS_INJECTION is true - "logging": config.logs_injection, + "logging": config.logs_injection, # type: ignore[attr-defined] "pynamodb": True, "pyodbc": True, "fastapi": True, diff --git a/ddtrace/provider.py b/ddtrace/provider.py index 7fac295351c..a556767c104 100644 --- a/ddtrace/provider.py +++ b/ddtrace/provider.py @@ -9,7 +9,7 @@ _DD_CONTEXTVAR = contextvars.ContextVar("datadog_contextvar", default=None) -class BaseContextProvider(six.with_metaclass(abc.ABCMeta)): +class BaseContextProvider(six.with_metaclass(abc.ABCMeta)): # type: ignore[misc] """ A ``ContextProvider`` is an interface that provides the blueprint for a callable class, capable to retrieve the current active diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index f64fd3cd8e7..1eed2115168 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -26,13 +26,13 @@ KNUTH_FACTOR = 1111111111111111111 -class BaseSampler(six.with_metaclass(abc.ABCMeta)): +class BaseSampler(six.with_metaclass(abc.ABCMeta)): # type: ignore[misc] @abc.abstractmethod def sample(self, span): pass -class BasePrioritySampler(six.with_metaclass(abc.ABCMeta)): +class BasePrioritySampler(six.with_metaclass(abc.ABCMeta)): # type: ignore[misc] @abc.abstractmethod def update_rate_by_service_sample_rates(self, sample_rates): pass diff --git a/mypy.ini b/mypy.ini index aa01f542970..df49b168f2c 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,5 @@ [mypy] -files = ddtrace/profiling +files = ddtrace/profiling, ddtrace/*.py show_error_codes = true # Not supported yet # exclude = ddtrace/vendors From aa4151ba3094aefcdbdf18cb7b9a0d0ceb156609 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Fri, 12 Mar 2021 19:10:31 -0500 Subject: [PATCH 03/15] Added type hinting to tracer, filters --- ddtrace/filters.py | 1 + ddtrace/internal/writer.py | 2 +- ddtrace/tracer.py | 101 +++++++++++++++++++------------------ 3 files changed, 54 insertions(+), 50 deletions(-) diff --git a/ddtrace/filters.py b/ddtrace/filters.py index 8d4e54f4c95..5d674bba8e2 100644 --- a/ddtrace/filters.py +++ b/ddtrace/filters.py @@ -52,6 +52,7 @@ def __init__(self, regexps): self._regexps = [re.compile(regexp) for regexp in regexps] def process_trace(self, trace): + # type: (List[Span]) -> Optional[List[Span]] """ When the filter is registered in the tracer, process_trace is called by on each trace before it is sent to the agent, the returned value will diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index ed9d0b71ef1..5416af453e1 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -159,7 +159,7 @@ def stop(self, timeout=None): return def write(self, spans): - # type: (List[Span]) -> None + # type: (Optional[List[Span]]) -> None if not spans: return diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index cba24d1c3e7..3d8617e59e6 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -4,10 +4,12 @@ from os import environ from os import getpid import sys -from typing import Dict +from typing import Any from typing import Callable +from typing import Dict from typing import List from typing import Optional +from typing import Set from typing import Union from ddtrace import config @@ -20,8 +22,8 @@ from .constants import SAMPLE_RATE_METRIC_KEY from .constants import VERSION_KEY from .context import Context -from .ext import system from .ext import SpanTypes +from .ext import system from .ext.priority import AUTO_KEEP from .ext.priority import AUTO_REJECT from .filters import TraceFilter @@ -37,6 +39,7 @@ from .internal.writer import AgentWriter from .internal.writer import LogWriter from .provider import DefaultContextProvider +from .sampler import BaseSampler from .sampler import DatadogSampler from .sampler import RateByServiceSampler from .sampler import RateSampler @@ -79,9 +82,9 @@ class Tracer(object): """ def __init__( - self, - url=None, # type: Optional[str] - dogstatsd_url=None # type: Optional[str] + self, + url=None, # type: Optional[str] + dogstatsd_url=None, # type: Optional[str] ): """ Create a new ``Tracer`` instance. A global tracer is already initialized @@ -91,7 +94,7 @@ def __init__( :param url: The DogStatsD URL. """ self.log = log - self.sampler = None + self.sampler = None # type: Optional[BaseSampler] self.priority_sampler = None self._runtime_worker = None self._filters = [] # type: List[TraceFilter] @@ -100,7 +103,7 @@ def __init__( self.tags = config.tags.copy() # a buffer for service info so we don't perpetually send the same things - self._services = set() + self._services = set() # type: Set[str] # Runtime id used for associating data collected during runtime to # traces @@ -185,24 +188,24 @@ async def web_handler(request): This method makes use of a ``ContextProvider`` that is automatically set during the tracer initialization, or while using a library instrumentation. """ - return self.context_provider.active(*args, **kwargs) + return self.context_provider.active(*args, **kwargs) # type: ignore # TODO: deprecate this method and make sure users create a new tracer if they need different parameters def configure( self, - enabled=None, - hostname=None, - port=None, - uds_path=None, - https=None, - sampler=None, - context_provider=None, - wrap_executor=None, - priority_sampling=None, - settings=None, - collect_metrics=None, - dogstatsd_url=None, - writer=None, + enabled=None, # type: Optional[bool] + hostname=None, # type: Optional[str] + port=None, # type: Optional[int] + uds_path=None, # type: Optional[str] + https=None, # type: Optional[bool] + sampler=None, # type: Optional[BaseSampler] + context_provider=None, # type: Optional[DefaultContextProvider] + wrap_executor=None, # type: Optional[Callable] + priority_sampling=None, # type: Optional[bool] + settings=None, # type: Optional[Dict[str, Any]] + collect_metrics=None, # type: Optional[bool] + dogstatsd_url=None, # type: Optional[str] + writer=None, # type: Optional[Union[AgentWriter, LogWriter]] ): """ Configure an existing Tracer the easy way. @@ -273,7 +276,7 @@ def configure( else: # No URL parts have updated and there's no previous writer to # get the URL from. - url = None + url = None # type: ignore self.writer.stop() if url: @@ -323,12 +326,12 @@ def configure( self._log_compat(logging.WARNING, msg) def start_span( - self, - name, # type: str - child_of=None, # type: Optional[Union[Span, Context]] - service=None, # type: Optional[str] - resource=None, # type: Optional[str] - span_type=None # type: Optional[Union[str, SpanTypes]] + self, + name, # type: str + child_of=None, # type: Optional[Union[Span, Context]] + service=None, # type: Optional[str] + resource=None, # type: Optional[str] + span_type=None, # type: Optional[Union[str, SpanTypes]] ): """ Return a span that will trace an operation called `name`. This method allows @@ -385,9 +388,9 @@ def start_span( if parent: service = parent.service else: - service = config.service # type: ignore[attr-defined] + service = config.service - mapped_service = config.service_mapping.get(service, service) # type: ignore[attr-defined] + mapped_service = config.service_mapping.get(service, service) if trace_id: # child_of a non-empty context, so either a local child span or from a remote context @@ -419,14 +422,14 @@ def start_span( ) span.metrics[system.PID] = self._pid or getpid() span.meta["runtime-id"] = get_runtime_id() - if config.report_hostname: # type: ignore[attr-defined] + if config.report_hostname: span.meta[HOSTNAME_KEY] = hostname.get_hostname() # add tags to root span to correlate trace with runtime metrics # only applied to spans with types that are internal to applications if self._runtime_worker and self._is_span_internal(span): span.meta["language"] = "python" - - span.sampled = self.sampler.sample(span) + # TODO: Can remove below type ignore once sampler is mypy type hinted + span.sampled = self.sampler.sample(span) # type: ignore[union-attr] # Old behavior # DEV: The new sampler sets metrics and priority sampling on the span for us if not isinstance(self.sampler, DatadogSampler): @@ -457,8 +460,8 @@ def start_span( if self.tags: span.set_tags(self.tags) - if config.env: # type: ignore[attr-defined] - span._set_str_tag(ENV_KEY, config.env) # type: ignore[attr-defined] + if config.env: + span._set_str_tag(ENV_KEY, config.env) # Only set the version tag on internal spans. if config.version: @@ -467,10 +470,10 @@ def start_span( # 2. the span is not the root, but the root span's service matches the span's service # and the root span has a version tag # then the span belongs to the user application and so set the version tag - if (root_span is None and service == config.service) or ( # type: ignore[attr-defined] + if (root_span is None and service == config.service) or ( root_span and root_span.service == service and VERSION_KEY in root_span.meta ): - span._set_str_tag(VERSION_KEY, config.version) # type: ignore[attr-defined] + span._set_str_tag(VERSION_KEY, config.version) # add it to the current context context.add_span(span) @@ -551,11 +554,11 @@ def _log_compat(self, level, msg): self.log.log(level, msg) def trace( - self, - name, # type: str - service=None, # type: Optional[str] - resource=None, # type: Optional[str] - span_type=None # type: Optional[Union[str, SpanTypes]] + self, + name, # type: str + service=None, # type: Optional[str] + resource=None, # type: Optional[str] + span_type=None, # type: Optional[Union[str, SpanTypes]] ): """ Return a span that will trace an operation called `name`. The context that created @@ -635,7 +638,7 @@ def current_span(self): return None def write(self, spans): - # type: (List[Span]) -> None + # type: (Optional[List[Span]]) -> None """ Send the trace to the writer to enqueue the spans list in the agent sending queue. @@ -651,7 +654,7 @@ def write(self, spans): if self.enabled and self.writer: for filtr in self._filters: try: - spans = filtr.process_trace(spans) + spans = filtr.process_trace(spans) # type: ignore[arg-type] except Exception: log.error("error while applying filter %s to traces", filtr, exc_info=True) else: @@ -666,11 +669,11 @@ def set_service_info(self, *args, **kwargs): return def wrap( - self, - name=None, # type: Optional[str] - service=None, # type: Optional[str] - resource=None, # type: Optional[str] - span_type=None # type: Optional[str] + self, + name=None, # type: Optional[str] + service=None, # type: Optional[str] + resource=None, # type: Optional[str] + span_type=None, # type: Optional[str] ): """ A decorator used to trace an entire function. If the traced function From e392434bbf9d3c0fb8ff7368c95d3cc56cd8132e Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Mon, 15 Mar 2021 12:39:21 -0400 Subject: [PATCH 04/15] Addressed PR reviews --- ddtrace/_hooks.py | 4 ++-- ddtrace/compat.py | 2 +- ddtrace/internal/_encoding.pyi | 4 +--- ddtrace/internal/_rand.pyi | 2 +- ddtrace/tracer.py | 5 ++--- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ddtrace/_hooks.py b/ddtrace/_hooks.py index a89497a1c10..5ef50cc0f84 100644 --- a/ddtrace/_hooks.py +++ b/ddtrace/_hooks.py @@ -1,6 +1,6 @@ import collections from copy import deepcopy -from typing import Dict +from typing import DefaultDict from typing import Set from .internal.logger import get_logger @@ -22,7 +22,7 @@ def on_request(span, request, response): pass """ - _hooks = attr.ib(init=False, factory=lambda: collections.defaultdict(set)) # type: Dict[str, Set] + _hooks = attr.ib(init=False, factory=lambda: collections.defaultdict(set), type=DefaultDict[str, Set]) def __deepcopy__(self, memodict=None): hooks = Hooks() diff --git a/ddtrace/compat.py b/ddtrace/compat.py index 76fc8a6bc1e..22d28dd9bf1 100644 --- a/ddtrace/compat.py +++ b/ddtrace/compat.py @@ -203,7 +203,7 @@ def get_connection_response(conn): try: import contextvars # noqa except ImportError: - from ddtrace.vendor import contextvars # type: ignore + from ddtrace.vendor import contextvars # type: ignore # noqa CONTEXTVARS_IS_AVAILABLE = False else: diff --git a/ddtrace/internal/_encoding.pyi b/ddtrace/internal/_encoding.pyi index 7fdb7331b33..a49baba15c1 100644 --- a/ddtrace/internal/_encoding.pyi +++ b/ddtrace/internal/_encoding.pyi @@ -1,8 +1,6 @@ from typing import Any from typing import Union - class MsgpackEncoder(object): content_type: str - - def _decode(self, data: Union[str, bytes]) -> Any: ... \ No newline at end of file + def _decode(self, data: Union[str, bytes]) -> Any: ... diff --git a/ddtrace/internal/_rand.pyi b/ddtrace/internal/_rand.pyi index 6e44c87b566..c8ecd497792 100644 --- a/ddtrace/internal/_rand.pyi +++ b/ddtrace/internal/_rand.pyi @@ -1,2 +1,2 @@ def seed() -> None: ... -def rand64bits(check_pid: bool = True) -> int: ... \ No newline at end of file +def rand64bits(check_pid: bool = True) -> int: ... diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 3d8617e59e6..44a169115d0 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -38,13 +38,12 @@ from .internal.runtime import get_runtime_id from .internal.writer import AgentWriter from .internal.writer import LogWriter +from .internal.writer import TraceWriter from .provider import DefaultContextProvider from .sampler import BaseSampler from .sampler import DatadogSampler from .sampler import RateByServiceSampler from .sampler import RateSampler - -# from .settings import config # TODO: type hint this bad boy OR just import directly from ddtrace from .span import Span from .utils.deprecation import deprecated from .utils.formats import asbool @@ -205,7 +204,7 @@ def configure( settings=None, # type: Optional[Dict[str, Any]] collect_metrics=None, # type: Optional[bool] dogstatsd_url=None, # type: Optional[str] - writer=None, # type: Optional[Union[AgentWriter, LogWriter]] + writer=None, # type: Optional[TraceWriter] ): """ Configure an existing Tracer the easy way. From 40f48d95e28e136c168ad68b4342c6504deef3e5 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Mon, 15 Mar 2021 14:41:46 -0400 Subject: [PATCH 05/15] Changed optional type for tracer.write and writer.write, added return type to wrap --- ddtrace/internal/writer.py | 2 +- ddtrace/tracer.py | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index 5416af453e1..ed9d0b71ef1 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -159,7 +159,7 @@ def stop(self, timeout=None): return def write(self, spans): - # type: (Optional[List[Span]]) -> None + # type: (List[Span]) -> None if not spans: return diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 44a169115d0..f8f94268586 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -160,6 +160,7 @@ def debug_logging(self): @debug_logging.setter # type: ignore[misc] @deprecated(message="Use logging.setLevel instead", version="1.0.0") def debug_logging(self, value): + # type: (bool) -> None self.log.setLevel(logging.DEBUG if value else logging.WARN) @deprecated("Use .tracer, not .tracer()", "1.0.0") @@ -637,7 +638,7 @@ def current_span(self): return None def write(self, spans): - # type: (Optional[List[Span]]) -> None + # type: (List[Span]) -> None """ Send the trace to the writer to enqueue the spans list in the agent sending queue. @@ -653,7 +654,7 @@ def write(self, spans): if self.enabled and self.writer: for filtr in self._filters: try: - spans = filtr.process_trace(spans) # type: ignore[arg-type] + spans = filtr.process_trace(spans) # type: ignore[arg-type, assignment] except Exception: log.error("error while applying filter %s to traces", filtr, exc_info=True) else: @@ -667,13 +668,8 @@ def set_service_info(self, *args, **kwargs): """Set the information about the given service.""" return - def wrap( - self, - name=None, # type: Optional[str] - service=None, # type: Optional[str] - resource=None, # type: Optional[str] - span_type=None, # type: Optional[str] - ): + def wrap(self, name=None, service=None, resource=None, span_type=None): + # type: (Optional[str], Optional[str], Optional[str], Optional[str]) -> Callable """ A decorator used to trace an entire function. If the traced function is a coroutine, it traces the coroutine execution when is awaited. From 51f43d964cf72a48d6e47e80425198c6aca037f8 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Mon, 15 Mar 2021 14:54:12 -0400 Subject: [PATCH 06/15] Added type hinting to compat.py, added return type hint to tracer.trace --- ddtrace/compat.py | 8 ++++++++ ddtrace/tracer.py | 9 ++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ddtrace/compat.py b/ddtrace/compat.py index 22d28dd9bf1..079bae313ca 100644 --- a/ddtrace/compat.py +++ b/ddtrace/compat.py @@ -4,9 +4,14 @@ import sys import textwrap import threading +from typing import Any +from typing import AnyStr +from typing import Text from ddtrace.vendor import six +from .internal.writer import Response + __all__ = [ "httplib", @@ -56,6 +61,7 @@ def is_integer(obj): + # type: (Any) -> bool """Helper to determine if the provided ``obj`` is an integer type or not""" # DEV: We have to make sure it is an integer and not a boolean # >>> type(True) @@ -164,6 +170,7 @@ def make_async_decorator(tracer, fn, *params, **kw_params): # type: ignore # DEV: There is `six.u()` which does something similar, but doesn't have the guard around `hasattr(s, 'decode')` def to_unicode(s): + # type: (AnyStr) -> Text """ Return a unicode string for the given bytes or string instance. """ # No reason to decode if we already have the unicode compatible object we expect # DEV: `six.text_type` will be a `str` for python 3 and `unicode` for python 2 @@ -183,6 +190,7 @@ def to_unicode(s): def get_connection_response(conn): + # type: (...) -> Response """Returns the response for a connection. If using Python 2 enable buffering. diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index f8f94268586..77604cc794c 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -553,13 +553,8 @@ def _log_compat(self, level, msg): else: self.log.log(level, msg) - def trace( - self, - name, # type: str - service=None, # type: Optional[str] - resource=None, # type: Optional[str] - span_type=None, # type: Optional[Union[str, SpanTypes]] - ): + def trace(self, name, service=None, resource=None, span_type=None): + # type: (str, Optional[str], Optional[str], Optional[Union[str, SpanTypes]]) -> Span """ Return a span that will trace an operation called `name`. The context that created the span as well as the span parenting, are automatically handled by the tracing From 66b13184b09895b7608c3ede2b1f78e61a2b9f3f Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Mon, 15 Mar 2021 18:00:51 -0400 Subject: [PATCH 07/15] Added return type hinting for wrap() --- ddtrace/tracer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 77604cc794c..5759b3ae417 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -664,7 +664,7 @@ def set_service_info(self, *args, **kwargs): return def wrap(self, name=None, service=None, resource=None, span_type=None): - # type: (Optional[str], Optional[str], Optional[str], Optional[str]) -> Callable + # type: (Optional[str], Optional[str], Optional[str], Optional[str]) -> Callable[[Callable[..., Any]], Callable[..., Any]] # noqa """ A decorator used to trace an entire function. If the traced function is a coroutine, it traces the coroutine execution when is awaited. From b439189e10fa6ea758393ce3bba62e01100b24a8 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Mon, 15 Mar 2021 18:21:28 -0400 Subject: [PATCH 08/15] Moved Span import to avoid circular import. --- ddtrace/internal/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index ed9d0b71ef1..5f2be5300c2 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -8,7 +8,6 @@ from typing import Optional import ddtrace -from ddtrace import Span from ddtrace.vendor import six from . import agent @@ -18,6 +17,7 @@ from ..constants import KEEP_SPANS_RATE_KEY from ..encoding import Encoder from ..encoding import JSONEncoderV2 +from ..span import Span from ..sampler import BasePrioritySampler from ..utils.time import StopWatch from .agent import get_connection From c8b9856ba5e909b8a920bfff0523b1663b55a477 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Mon, 15 Mar 2021 18:25:43 -0400 Subject: [PATCH 09/15] Fixed flake8 --- ddtrace/internal/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index 5f2be5300c2..eee0387f685 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -17,8 +17,8 @@ from ..constants import KEEP_SPANS_RATE_KEY from ..encoding import Encoder from ..encoding import JSONEncoderV2 -from ..span import Span from ..sampler import BasePrioritySampler +from ..span import Span from ..utils.time import StopWatch from .agent import get_connection from .buffer import BufferFull From 38fe25597cb83af6a2b1a67179f49e88fa736607 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Mon, 15 Mar 2021 18:31:53 -0400 Subject: [PATCH 10/15] Removed circular import from compat.py, reverted previous change --- ddtrace/compat.py | 4 +--- ddtrace/internal/writer.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ddtrace/compat.py b/ddtrace/compat.py index 079bae313ca..ced42e58da8 100644 --- a/ddtrace/compat.py +++ b/ddtrace/compat.py @@ -8,10 +8,9 @@ from typing import AnyStr from typing import Text +# from ddtrace.internal.writer import Response from ddtrace.vendor import six -from .internal.writer import Response - __all__ = [ "httplib", @@ -190,7 +189,6 @@ def to_unicode(s): def get_connection_response(conn): - # type: (...) -> Response """Returns the response for a connection. If using Python 2 enable buffering. diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index eee0387f685..b7b2f3e92b4 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -9,6 +9,7 @@ import ddtrace from ddtrace.vendor import six +from ddtrace import Span from . import agent from .. import _worker @@ -18,7 +19,6 @@ from ..encoding import Encoder from ..encoding import JSONEncoderV2 from ..sampler import BasePrioritySampler -from ..span import Span from ..utils.time import StopWatch from .agent import get_connection from .buffer import BufferFull From d04dc2cba277344d23d300fbedc1d8d1bf29f018 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Mon, 15 Mar 2021 18:34:22 -0400 Subject: [PATCH 11/15] Flake8 reformatting --- ddtrace/internal/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index b7b2f3e92b4..ed9d0b71ef1 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -8,8 +8,8 @@ from typing import Optional import ddtrace -from ddtrace.vendor import six from ddtrace import Span +from ddtrace.vendor import six from . import agent from .. import _worker From eb8f842803c41d14c7a984e40c67ac38692b1ed9 Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Tue, 16 Mar 2021 10:38:59 -0400 Subject: [PATCH 12/15] Address PR comments --- ddtrace/compat.py | 1 - ddtrace/tracer.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/compat.py b/ddtrace/compat.py index ced42e58da8..54ed0885ae4 100644 --- a/ddtrace/compat.py +++ b/ddtrace/compat.py @@ -8,7 +8,6 @@ from typing import AnyStr from typing import Text -# from ddtrace.internal.writer import Response from ddtrace.vendor import six diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 34f186248eb..86423e98f96 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -86,6 +86,7 @@ def __init__( url=None, # type: Optional[str] dogstatsd_url=None, # type: Optional[str] ): + # type: (...) -> None """ Create a new ``Tracer`` instance. A global tracer is already initialized for common usage, so there is no need to initialize your own ``Tracer``. From 8eb130a42d204e098a17e0d9aa3248d56d270f9d Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Tue, 16 Mar 2021 11:08:43 -0400 Subject: [PATCH 13/15] Reverted tracer.write and writer.write to accept Optional[List[Span]] --- ddtrace/internal/writer.py | 4 ++-- ddtrace/tracer.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ddtrace/internal/writer.py b/ddtrace/internal/writer.py index ed9d0b71ef1..48b4908b4e6 100644 --- a/ddtrace/internal/writer.py +++ b/ddtrace/internal/writer.py @@ -133,7 +133,7 @@ def stop(self, timeout=None): @abc.abstractmethod def write(self, trace): - # type: (List[Span]) -> None + # type: (Optional[List[Span]]) -> None pass @@ -159,7 +159,7 @@ def stop(self, timeout=None): return def write(self, spans): - # type: (List[Span]) -> None + # type: (Optional[List[Span]]) -> None if not spans: return diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index 86423e98f96..b36391a12be 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -129,7 +129,7 @@ def __init__( report_metrics=config.health_metrics_enabled, ) self.writer = writer - self.processor = TraceProcessor([]) + self.processor = TraceProcessor([]) # type: ignore[call-arg] self._hooks = _hooks.Hooks() def on_start_span(self, func): @@ -292,7 +292,7 @@ def configure( report_metrics=config.health_metrics_enabled, ) self.writer.dogstatsd = get_dogstatsd_client(self._dogstatsd_url) - self.processor = TraceProcessor(filters=self._filters) + self.processor = TraceProcessor(filters=self._filters) # type: ignore[call-arg] if context_provider is not None: self.context_provider = context_provider @@ -637,7 +637,7 @@ def current_span(self): return None def write(self, spans): - # type: (List[Span]) -> None + # type: (Optional[List[Span]]) -> None """ Send the trace to the writer to enqueue the spans list in the agent sending queue. From d7894fc5ae17231c338dece54e960e594178d78e Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Tue, 16 Mar 2021 11:35:45 -0400 Subject: [PATCH 14/15] Added return type hinting --- ddtrace/tracer.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index b36391a12be..f52e0d325e9 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -210,6 +210,7 @@ def configure( dogstatsd_url=None, # type: Optional[str] writer=None, # type: Optional[TraceWriter] ): + # type: (...) -> None """ Configure an existing Tracer the easy way. Allow to configure or reconfigure a Tracer instance. @@ -337,6 +338,7 @@ def start_span( resource=None, # type: Optional[str] span_type=None, # type: Optional[Union[str, SpanTypes]] ): + # type: (...) -> Span """ Return a span that will trace an operation called `name`. This method allows parenting using the ``child_of`` kwarg. If it's missing, the newly created span is a @@ -662,8 +664,14 @@ def set_service_info(self, *args, **kwargs): """Set the information about the given service.""" return - def wrap(self, name=None, service=None, resource=None, span_type=None): - # type: (Optional[str], Optional[str], Optional[str], Optional[str]) -> Callable[[Callable[..., Any]], Callable[..., Any]] # noqa + def wrap( + self, + name=None, # type: Optional[str] + service=None, # type: Optional[str] + resource=None, # type: Optional[str] + span_type=None # type: Optional[str] + ): + # type: (...) -> Callable[[Callable[..., Any]], Callable[..., Any]] """ A decorator used to trace an entire function. If the traced function is a coroutine, it traces the coroutine execution when is awaited. From b18f7d0c2d8aae343fd7c128d8fb09cb8b95238a Mon Sep 17 00:00:00 2001 From: Yun Kim Date: Tue, 16 Mar 2021 11:45:27 -0400 Subject: [PATCH 15/15] Black reformatting --- ddtrace/compat.py | 2 +- ddtrace/monkey.py | 2 +- ddtrace/tracer.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ddtrace/compat.py b/ddtrace/compat.py index 54ed0885ae4..c3ea37f0b03 100644 --- a/ddtrace/compat.py +++ b/ddtrace/compat.py @@ -162,7 +162,7 @@ def func_wrapper(*args, **kwargs): def iscoroutinefunction(fn): # type: ignore return False - def make_async_decorator(tracer, fn, *params, **kw_params): # type: ignore + def make_async_decorator(tracer, fn, *params, **kw_params): return fn diff --git a/ddtrace/monkey.py b/ddtrace/monkey.py index 31cb50d281b..66fa8d81926 100644 --- a/ddtrace/monkey.py +++ b/ddtrace/monkey.py @@ -64,7 +64,7 @@ "pylons": False, "pyramid": False, # Auto-enable logging if the environment variable DD_LOGS_INJECTION is true - "logging": config.logs_injection, # type: ignore[attr-defined] + "logging": config.logs_injection, "pynamodb": True, "pyodbc": True, "fastapi": True, diff --git a/ddtrace/tracer.py b/ddtrace/tracer.py index f52e0d325e9..96d3bc322ca 100644 --- a/ddtrace/tracer.py +++ b/ddtrace/tracer.py @@ -665,11 +665,11 @@ def set_service_info(self, *args, **kwargs): return def wrap( - self, - name=None, # type: Optional[str] - service=None, # type: Optional[str] - resource=None, # type: Optional[str] - span_type=None # type: Optional[str] + self, + name=None, # type: Optional[str] + service=None, # type: Optional[str] + resource=None, # type: Optional[str] + span_type=None, # type: Optional[str] ): # type: (...) -> Callable[[Callable[..., Any]], Callable[..., Any]] """