fix(boto3): Trace the complete botocore client-call lifecycle (#7538)

### Description

Move boto3 span creation from individual HTTP request attempts to full
botocore client-call lifecycle.

Previously, the client span was created from the botocore
`request-created` event; since botocore creates a new `AwsRequest` event
for every retry, these spans represent individual retries rather than
the full event. Following OTel
([https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span](<https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span>))
the span should cover the entire call lifecycle including all retries.
Wrapping `_make_api_call()` covers all retries performed by botocore,
including serialization, endpoint resolution, the final response or
failures
([https://github.com/boto/botocore/blob/develop/botocore/client.py:999](<https://github.com/boto/botocore/blob/develop/botocore/client.py:999>)).

* patch `BaseClient._make_api_call()` so one span represents a single
boto operation; this span is also kept active across every retry
attempt.
* `request-created` is kept for breadcrumbs, HTTP request attributes,
and trace propagation.
* client spans that fail will preserve the original exception.
* responses that return a `StreamingBody` are kept open until body is
consumed or closed.
* fix the correct parent/child relationship between boto, HTTP, and
streaming spans.<br><br>

#### Issues

Resolves #7474

Author: pabloDeputter

sentry_sdk/consts.py

@@ -1172,6 +1172,18 @@ class SPANDATA:
     Used in inbound filters.
     """
 
+    SENTRY_OP = "sentry.op"
+    """
+    The operation of a span.
+    Example: "http.client"
+    """
+
+    SENTRY_ORIGIN = "sentry.origin"
+    """
+    The origin of the instrumentation (e.g. span, log, etc.)
+    Example: "auto.http.otel.fastify"
+    """
+
     SENTRY_RELEASE = "sentry.release"
     """
     The Sentry release.

sentry_sdk/integrations/boto3/_client.py

@@ -1,37 +1,127 @@
-from functools import partial
+from contextlib import contextmanager
 from typing import TYPE_CHECKING
 
+import sentry_sdk
 from sentry_sdk.integrations import DidNotEnable
+from sentry_sdk.integrations.boto3._context import AwsCallContext
 from sentry_sdk.integrations.boto3._instrumentation import (
-    _sentry_after_call,
-    _sentry_after_call_error,
+    _finish_span,
+    _instrument_streaming_body,
     _sentry_before_sign,
     _sentry_request_created,
+    _start_client_span,
 )
+from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan
+from sentry_sdk.utils import capture_internal_exceptions
 
 if TYPE_CHECKING:
-    from typing import Any
+    from typing import Any, Iterator, Optional, Union
+
+    from sentry_sdk.tracing import Span
 
 try:
     from botocore.client import BaseClient
 except ImportError:
-    raise DidNotEnable("botocore is not installed")
+    raise DidNotEnable("botocore not installed")
+
+
+@contextmanager
+def _activate_client_span(
+    span: "Union[Span, StreamedSpan]",
+) -> "Iterator[Union[Span, StreamedSpan]]":
+    """
+    Activate the boto span temporarily during `_make_api_call()` without ending it.
+
+    Botocore returns a `StreamingBody` before its bytes are consumed. Using the
+    context manager would finish it as soon as `_make_api_call()` returns, so
+    restore the caller's span here and let the `StreamingBody` wrapper finish
+    the boto span when body is consumed/closed.
+
+    faulty:                               desired:
+           boto3  [_make_api_call]                boto3  [_make_api_call------]
+           http     [request]                     http       [request]
+           stream               [read]            stream                [read]
+    """
+    if isinstance(span, NoOpStreamedSpan):
+        yield span
+        return
+
+    scope = sentry_sdk.get_current_scope()
+    if not isinstance(span, StreamedSpan):
+        previous_span = scope.span
+        scope.span = span
+        try:
+            yield span
+        finally:
+            scope.span = previous_span
+        return
+
+    previous_streamed_span = scope.streamed_span
+    scope.streamed_span = span
+    try:
+        yield span
+    finally:
+        scope.streamed_span = previous_streamed_span
 
 
 def _patch_botocore_client() -> None:
+    from sentry_sdk.integrations.boto3 import Boto3Integration
+
     orig_init = BaseClient.__init__
+    orig_make_api_call = BaseClient._make_api_call  # type: ignore
 
     def sentry_patched_init(self: "BaseClient", *args: "Any", **kwargs: "Any") -> None:
         orig_init(self, *args, **kwargs)
         meta = self.meta
-        service_id = meta.service_model.service_id
-        meta.events.register(
-            "request-created",
-            partial(_sentry_request_created, service_id=service_id),
-        )
-        # run after other `before-sign` handlers, allowing it to see and preserve existing baggage.
+        meta.events.register("request-created", _sentry_request_created)
+        # run after other `before-sign` handlers so existing baggage is preserved.
         meta.events.register_last("before-sign", _sentry_before_sign)
-        meta.events.register("after-call", _sentry_after_call)
-        meta.events.register("after-call-error", _sentry_after_call_error)
+
+    def sentry_patched_make_api_call(
+        self: "BaseClient", operation_name: str, api_params: "Any"
+    ) -> "Any":
+        """
+        Track a single API call, including retries, serialization, and endpoint
+        resolution. For streaming responses, keep the span open until the
+        response body is consumed or closed.
+        https://github.com/boto/botocore/blob/358f8eec8c76201bb1a7a35644abcbc9036de7ed/botocore/client.py
+        https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span
+        """
+        client = sentry_sdk.get_client()
+        if client.get_integration(Boto3Integration) is None:
+            return orig_make_api_call(self, operation_name, api_params)
+
+        ctx = AwsCallContext(operation_name)
+
+        # add optional metadata to context.
+        with capture_internal_exceptions():
+            ctx.add_metadata(self)
+
+        span: "Optional[Union[Span, StreamedSpan]]" = None
+        with capture_internal_exceptions():
+            span = _start_client_span(ctx)
+
+        if span is None:
+            return orig_make_api_call(self, operation_name, api_params)
+
+        # activate without finishing; a streaming response may outlive the call.
+        span_ctx = _activate_client_span(span)
+
+        try:
+            with span_ctx:
+                parsed = orig_make_api_call(self, operation_name, api_params)
+        except BaseException as error:
+            _finish_span(span, error)
+            raise
+
+        streaming_body_instrumented = False
+        with capture_internal_exceptions():
+            streaming_body_instrumented = _instrument_streaming_body(span, parsed)
+
+        # `StreamingBody`s finish their span when consumed or closed.
+        if not streaming_body_instrumented:
+            _finish_span(span)
+        return parsed
 
     BaseClient.__init__ = sentry_patched_init  # type: ignore
+    BaseClient._make_api_call = sentry_patched_make_api_call  # type: ignore

sentry_sdk/integrations/boto3/_context.py

@@ -0,0 +1,44 @@
+from typing import TYPE_CHECKING
+
+from sentry_sdk.integrations import DidNotEnable
+from sentry_sdk.utils import capture_internal_exceptions
+
+if TYPE_CHECKING:
+    from typing import Any, Optional
+
+try:
+    from botocore.client import BaseClient
+except ImportError:
+    raise DidNotEnable("botocore not installed")
+
+
+class AwsCallContext:
+    __slots__ = (
+        "service_id",
+        "service_id_hyphenized",
+        "operation_name",
+    )
+
+    def __init__(self, operation_name: str) -> None:
+        self.operation_name: str = operation_name
+        self.service_id: "Optional[str]" = None
+        self.service_id_hyphenized: "Optional[str]" = None
+
+    def add_metadata(self, client: "BaseClient") -> None:
+        def _get_attr(obj: "Any", name: str) -> "Any":
+            if obj is None:
+                return None
+
+            with capture_internal_exceptions():
+                return getattr(obj, name)
+
+        client_meta = _get_attr(client, "meta")
+        service_model = _get_attr(client_meta, "service_model")
+
+        # modeled AWS service identity used in span names, e.g. `API Gateway`.
+        service_id = _get_attr(service_model, "service_id")
+        if service_id is not None:
+            with capture_internal_exceptions():
+                self.service_id = str(service_id)
+            with capture_internal_exceptions():
+                self.service_id_hyphenized = service_id.hyphenize()