@@ -48,9 +48,7 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
4848/// work before the boundary begins listening. Keep this aligned with the
4949/// driver bootstrap grace period rather than the normal operation timeout.
5050const ATTACH_REQUEST_TIMEOUT : Duration = Duration :: from_mins ( 5 ) ;
51- /// How long one control call keeps retrying boundary connect attempts. Boot-time
52- /// callers retry whole calls above this; past boot, exhausting this window
53- /// means the remote boundary (or its launcher) is gone rather than still starting.
51+ /// How long to wait for a replacement boundary connection after first contact.
5452const CONNECT_RETRY_TIMEOUT : Duration = Duration :: from_secs ( 30 ) ;
5553/// Bounds the liveness check that decides whether a stream failure lost the
5654/// whole connection.
@@ -132,6 +130,21 @@ enum TransportSource {
132130 Routed ( Arc < crate :: runtime_routed:: RouteRegistration > ) ,
133131}
134132
133+ struct BoundaryConnectAttempts {
134+ span : tracing:: Span ,
135+ count : u64 ,
136+ }
137+
138+ impl Drop for BoundaryConnectAttempts {
139+ fn drop ( & mut self ) {
140+ // Record the count once if any attempts began, including on cancellation.
141+ if self . count > 0 {
142+ self . span
143+ . record ( "openshell.boundary.connect_attempts" , self . count ) ;
144+ }
145+ }
146+ }
147+
135148impl BoundaryTransport {
136149 /// Select the transport the descriptor requires. A runtime-initiated
137150 /// descriptor binds its control listener, so call this inside the
@@ -179,46 +192,67 @@ impl BoundaryTransport {
179192
180193 /// Open one authenticated physical connection, retrying within the
181194 /// connect window while the remote boundary starts.
195+ #[ tracing:: instrument(
196+ name = "supervisor.boundary.connect" ,
197+ skip_all,
198+ fields( openshell. boundary. transport = match & self . 0 {
199+ TransportSource :: Dial => "dial" ,
200+ TransportSource :: Connector ( _) => "backend" ,
201+ TransportSource :: RuntimeInitiated ( _) => "runtime_initiated" ,
202+ TransportSource :: Routed ( _) => "routed" ,
203+ } , openshell. boundary. connect_attempts = tracing:: field:: Empty , otel. status_code = tracing:: field:: Empty )
204+ ) ]
182205 async fn connect (
183206 & self ,
184207 descriptor : & SandboxRuntimeDescriptor ,
185- initial_connect_timeout : Duration ,
208+ connect_timeout : Duration ,
186209 ) -> Result < BoundaryDuplexStream , BackendError > {
187- let connector = match & self . 0 {
188- TransportSource :: Dial => None ,
189- TransportSource :: Connector ( connector) => Some ( connector. as_ref ( ) ) ,
190- TransportSource :: RuntimeInitiated ( listener) => {
191- return tokio:: time:: timeout ( CONNECT_RETRY_TIMEOUT , listener. accept ( descriptor) )
192- . await
193- . map_err ( |_| {
194- BackendError :: Unavailable (
195- "timed out waiting for the sandbox runtime to connect" . to_string ( ) ,
196- )
197- } ) ?;
198- }
199- TransportSource :: Routed ( registration) => {
200- let window = if registration. has_connected ( ) {
201- CONNECT_RETRY_TIMEOUT
202- } else {
203- initial_connect_timeout
204- } ;
205- return tokio:: time:: timeout ( window, registration. accept ( & descriptor. tls ) )
210+ let mut attempts = BoundaryConnectAttempts {
211+ span : tracing:: Span :: current ( ) ,
212+ count : 0 ,
213+ } ;
214+ let result = async {
215+ let connector = match & self . 0 {
216+ TransportSource :: Dial => None ,
217+ TransportSource :: Connector ( connector) => Some ( connector. as_ref ( ) ) ,
218+ TransportSource :: RuntimeInitiated ( listener) => {
219+ return tokio:: time:: timeout ( connect_timeout, listener. accept ( descriptor) )
220+ . await
221+ . map_err ( |_| {
222+ BackendError :: Unavailable (
223+ "timed out waiting for the sandbox runtime to connect" . to_string ( ) ,
224+ )
225+ } ) ?;
226+ }
227+ TransportSource :: Routed ( registration) => {
228+ return tokio:: time:: timeout (
229+ connect_timeout,
230+ registration. accept ( & descriptor. tls ) ,
231+ )
206232 . await
207233 . map_err ( |_| {
208234 BackendError :: Unavailable (
209235 "timed out waiting for the sandbox runtime to connect" . to_string ( ) ,
210236 )
211237 } ) ?;
212- }
213- } ;
214- let deadline = tokio:: time:: Instant :: now ( ) + CONNECT_RETRY_TIMEOUT ;
215- loop {
216- match connect_boundary_using_connector ( descriptor, connector) . await {
217- Ok ( stream) => return Ok ( stream) ,
218- Err ( error) if tokio:: time:: Instant :: now ( ) >= deadline => return Err ( error) ,
219- Err ( _) => tokio:: time:: sleep ( Duration :: from_millis ( 25 ) ) . await ,
238+ }
239+ } ;
240+ let deadline = tokio:: time:: Instant :: now ( ) + connect_timeout;
241+ loop {
242+ attempts. count += 1 ;
243+ match connect_boundary_using_connector ( descriptor, connector) . await {
244+ Ok ( stream) => return Ok ( stream) ,
245+ Err ( error) if tokio:: time:: Instant :: now ( ) >= deadline => return Err ( error) ,
246+ Err ( _) => tokio:: time:: sleep ( Duration :: from_millis ( 25 ) ) . await ,
247+ }
220248 }
221249 }
250+ . await ;
251+ attempts. span . record (
252+ "otel.status_code" ,
253+ if result. is_ok ( ) { "OK" } else { "ERROR" } ,
254+ ) ;
255+ result
222256 }
223257}
224258
@@ -2334,7 +2368,9 @@ impl BoundaryClient {
23342368 confirm : Option < & RequestEnvelope > ,
23352369 credential : & BoundaryCredential ,
23362370 ) -> Result < ( tonic:: transport:: Channel , Arc < TransportAbort > ) , BackendError > {
2337- let ( channel, transport) = self . build_grpc_channel ( credential) . await ?;
2371+ let ( channel, transport) = self
2372+ . build_grpc_channel ( credential, CONNECT_RETRY_TIMEOUT )
2373+ . await ?;
23382374 let result = async {
23392375 self . exchange_on_channel ( channel. clone ( ) , attach, credential)
23402376 . await ?;
@@ -2494,7 +2530,11 @@ impl BoundaryClient {
24942530 return Ok ( cached. channel . clone ( ) ) ;
24952531 }
24962532 let credential = BoundaryCredential :: capture ( & self . sandbox_bearer ) ?;
2497- let ( channel, transport) = self . build_grpc_channel ( & credential) . await ?;
2533+ // Initial policy discovery and attach already have a startup deadline.
2534+ // Keep the transport wait in one connection span within that budget.
2535+ let ( channel, transport) = self
2536+ . build_grpc_channel ( & credential, self . attach_request_timeout )
2537+ . await ?;
24982538 * state = Some ( CachedGrpcChannel {
24992539 credential_epoch : credential. epoch ,
25002540 bearer_fingerprint : credential. fingerprint ,
@@ -2510,10 +2550,10 @@ impl BoundaryClient {
25102550 async fn build_grpc_channel (
25112551 & self ,
25122552 credential : & BoundaryCredential ,
2553+ connect_timeout : Duration ,
25132554 ) -> Result < ( tonic:: transport:: Channel , Arc < TransportAbort > ) , BackendError > {
25142555 let runtime_descriptor = self . runtime_descriptor . clone ( ) ;
25152556 let source = self . transport . clone ( ) ;
2516- let initial_connect_timeout = self . attach_request_timeout ;
25172557 let transport = Arc :: new ( TransportAbort :: default ( ) ) ;
25182558 let connector_transport = transport. clone ( ) ;
25192559 let authority = match & runtime_descriptor. transport {
@@ -2547,7 +2587,7 @@ impl BoundaryClient {
25472587 return Err ( AbortableTransport :: aborted_error ( ) ) ;
25482588 }
25492589 source
2550- . connect ( & runtime_descriptor, initial_connect_timeout )
2590+ . connect ( & runtime_descriptor, connect_timeout )
25512591 . await
25522592 . map ( |inner| TokioIo :: new ( AbortableTransport { inner, abort } ) )
25532593 . map_err ( |error| std:: io:: Error :: other ( error. to_string ( ) ) )
@@ -3094,6 +3134,7 @@ pub mod test_support;
30943134
30953135#[ cfg( test) ]
30963136mod tests {
3137+ mod connection_tracing;
30973138 mod credential_renewal;
30983139 mod delegated;
30993140 mod flow_control;
@@ -4449,7 +4490,7 @@ mod tests {
44494490 }
44504491
44514492 #[ tokio:: test( start_paused = true ) ]
4452- async fn configured_routed_bootstrap_wait_preserves_reconnect_window ( ) {
4493+ async fn routed_connection_uses_the_callers_timeout ( ) {
44534494 let certificate = test_certificate ( ) ;
44544495 let mut descriptor =
44554496 tls_runtime_descriptor ( "127.0.0.1:1" . parse ( ) . unwrap ( ) , certificate. client_tls ) ;
@@ -4461,7 +4502,7 @@ mod tests {
44614502 route : route. clone ( ) ,
44624503 } ;
44634504 let registration = router. register ( & route, & descriptor. tls ) . unwrap ( ) ;
4464- let transport = BoundaryTransport :: routed ( & descriptor, registration. clone ( ) ) . unwrap ( ) ;
4505+ let transport = BoundaryTransport :: routed ( & descriptor, registration) . unwrap ( ) ;
44654506 let budget = Duration :: from_mins ( 15 ) ;
44664507 let mut waiting = Box :: pin ( transport. connect ( & descriptor, budget) ) ;
44674508 poll_fn ( |context| {
@@ -4500,12 +4541,28 @@ mod tests {
45004541 . unwrap ( )
45014542 . unwrap ( ) ;
45024543 drop ( waiting) ;
4503- assert ! ( registration. has_connected( ) ) ;
45044544 drop ( stream) ;
45054545 drop ( runtime. await . unwrap ( ) ) ;
45064546
45074547 tokio:: time:: pause ( ) ;
4508- let mut reconnect = Box :: pin ( transport. connect ( & descriptor, budget) ) ;
4548+ let mut attach = Box :: pin ( transport. connect ( & descriptor, budget) ) ;
4549+ poll_fn ( |context| {
4550+ assert ! ( attach. as_mut( ) . poll( context) . is_pending( ) ) ;
4551+ Poll :: Ready ( ( ) )
4552+ } )
4553+ . await ;
4554+ tokio:: time:: advance ( CONNECT_RETRY_TIMEOUT + Duration :: from_secs ( 1 ) ) . await ;
4555+ poll_fn ( |context| {
4556+ assert ! (
4557+ attach. as_mut( ) . poll( context) . is_pending( ) ,
4558+ "a previous connection must not shorten the startup budget for attach"
4559+ ) ;
4560+ Poll :: Ready ( ( ) )
4561+ } )
4562+ . await ;
4563+ drop ( attach) ;
4564+
4565+ let mut reconnect = Box :: pin ( transport. connect ( & descriptor, CONNECT_RETRY_TIMEOUT ) ) ;
45094566 poll_fn ( |context| {
45104567 assert ! ( reconnect. as_mut( ) . poll( context) . is_pending( ) ) ;
45114568 Poll :: Ready ( ( ) )
0 commit comments