Skip to content

Commit 498a0bb

Browse files
shiju-nvkrishicks
andauthored
feat(tracing): show boundary connection time during sandbox startup (#5243)
* feat(tracing): show boundary connection time during sandbox startup Signed-off-by: Shiju <shiju@nvidia.com> * fix(tracing): report boundary connection status and attempts Signed-off-by: Shiju <shiju@nvidia.com> * fix(tracing): record boundary attempt count once Recording the attempt count on every retry appends duplicate OTLP attributes. A connection that succeeds on the third attempt exports four entries for the same key. A long retry loop can consume the span attribute budget before the final count is recorded. Track attempts for the lifetime of the connection future and record the count once when it ends, including on cancellation. Verify the exported span with a loopback OTLP collector, since the existing map-based test replaces earlier values and hides duplicates. Signed-off-by: Kris Hicks <khicks@nvidia.com> * fix(tracing): keep boundary startup waits in one span Policy discovery can outlast the 30-second connection retry window while the sandbox starts. Restarting the connection span at each window adds ERROR spans to successful startups and splits the wait operators need to measure. Use the configured startup budget for the initial connection and keep the 30-second window for reattachment. Let the caller choose that budget on every transport so sharing a routed registration between discovery and attach does not shorten the attach window. Emit the attempt count only when the supervisor dials or invokes a backend connector. Inbound transports wait for an authenticated stream and cannot count the runtime’s dials. If the startup deadline cancels the wait, the startup span records the failure. Signed-off-by: Kris Hicks <khicks@nvidia.com> --------- Signed-off-by: Shiju <shiju@nvidia.com> Signed-off-by: Kris Hicks <khicks@nvidia.com> Co-authored-by: Kris Hicks <khicks@nvidia.com>
1 parent c24fcd2 commit 498a0bb

7 files changed

Lines changed: 599 additions & 54 deletions

File tree

‎Cargo.lock‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/openshell-sandbox-backend/Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,11 @@ libc = "0.2"
4343
rustix = { workspace = true, features = ["fs", "process"] }
4444

4545
[dev-dependencies]
46+
openshell-otel = { path = "../openshell-otel" }
47+
openshell-otel-test-support = { path = "../openshell-otel-test-support" }
4648
tokio = { workspace = true, features = ["test-util"] }
4749
jsonwebtoken = { workspace = true }
50+
tracing-subscriber = { workspace = true }
4851

4952
[lints]
5053
workspace = true

‎crates/openshell-sandbox-backend/src/runtime.rs‎

Lines changed: 96 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,7 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
4848
/// work before the boundary begins listening. Keep this aligned with the
4949
/// driver bootstrap grace period rather than the normal operation timeout.
5050
const ATTACH_REQUEST_TIMEOUT: Duration = Duration::from_mins(5);
51-
/// How long one control call keeps retrying boundary connect attempts. Boot-time
52-
/// callers retry whole calls above this; past boot, exhausting this window
53-
/// means the remote boundary (or its launcher) is gone rather than still starting.
51+
/// How long to wait for a replacement boundary connection after first contact.
5452
const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30);
5553
/// Bounds the liveness check that decides whether a stream failure lost the
5654
/// whole connection.
@@ -132,6 +130,21 @@ enum TransportSource {
132130
Routed(Arc<crate::runtime_routed::RouteRegistration>),
133131
}
134132

133+
struct BoundaryConnectAttempts {
134+
span: tracing::Span,
135+
count: u64,
136+
}
137+
138+
impl Drop for BoundaryConnectAttempts {
139+
fn drop(&mut self) {
140+
// Record the count once if any attempts began, including on cancellation.
141+
if self.count > 0 {
142+
self.span
143+
.record("openshell.boundary.connect_attempts", self.count);
144+
}
145+
}
146+
}
147+
135148
impl BoundaryTransport {
136149
/// Select the transport the descriptor requires. A runtime-initiated
137150
/// descriptor binds its control listener, so call this inside the
@@ -179,46 +192,67 @@ impl BoundaryTransport {
179192

180193
/// Open one authenticated physical connection, retrying within the
181194
/// connect window while the remote boundary starts.
195+
#[tracing::instrument(
196+
name = "supervisor.boundary.connect",
197+
skip_all,
198+
fields(openshell.boundary.transport = match &self.0 {
199+
TransportSource::Dial => "dial",
200+
TransportSource::Connector(_) => "backend",
201+
TransportSource::RuntimeInitiated(_) => "runtime_initiated",
202+
TransportSource::Routed(_) => "routed",
203+
}, openshell.boundary.connect_attempts = tracing::field::Empty, otel.status_code = tracing::field::Empty)
204+
)]
182205
async fn connect(
183206
&self,
184207
descriptor: &SandboxRuntimeDescriptor,
185-
initial_connect_timeout: Duration,
208+
connect_timeout: Duration,
186209
) -> Result<BoundaryDuplexStream, BackendError> {
187-
let connector = match &self.0 {
188-
TransportSource::Dial => None,
189-
TransportSource::Connector(connector) => Some(connector.as_ref()),
190-
TransportSource::RuntimeInitiated(listener) => {
191-
return tokio::time::timeout(CONNECT_RETRY_TIMEOUT, listener.accept(descriptor))
192-
.await
193-
.map_err(|_| {
194-
BackendError::Unavailable(
195-
"timed out waiting for the sandbox runtime to connect".to_string(),
196-
)
197-
})?;
198-
}
199-
TransportSource::Routed(registration) => {
200-
let window = if registration.has_connected() {
201-
CONNECT_RETRY_TIMEOUT
202-
} else {
203-
initial_connect_timeout
204-
};
205-
return tokio::time::timeout(window, registration.accept(&descriptor.tls))
210+
let mut attempts = BoundaryConnectAttempts {
211+
span: tracing::Span::current(),
212+
count: 0,
213+
};
214+
let result = async {
215+
let connector = match &self.0 {
216+
TransportSource::Dial => None,
217+
TransportSource::Connector(connector) => Some(connector.as_ref()),
218+
TransportSource::RuntimeInitiated(listener) => {
219+
return tokio::time::timeout(connect_timeout, listener.accept(descriptor))
220+
.await
221+
.map_err(|_| {
222+
BackendError::Unavailable(
223+
"timed out waiting for the sandbox runtime to connect".to_string(),
224+
)
225+
})?;
226+
}
227+
TransportSource::Routed(registration) => {
228+
return tokio::time::timeout(
229+
connect_timeout,
230+
registration.accept(&descriptor.tls),
231+
)
206232
.await
207233
.map_err(|_| {
208234
BackendError::Unavailable(
209235
"timed out waiting for the sandbox runtime to connect".to_string(),
210236
)
211237
})?;
212-
}
213-
};
214-
let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT;
215-
loop {
216-
match connect_boundary_using_connector(descriptor, connector).await {
217-
Ok(stream) => return Ok(stream),
218-
Err(error) if tokio::time::Instant::now() >= deadline => return Err(error),
219-
Err(_) => tokio::time::sleep(Duration::from_millis(25)).await,
238+
}
239+
};
240+
let deadline = tokio::time::Instant::now() + connect_timeout;
241+
loop {
242+
attempts.count += 1;
243+
match connect_boundary_using_connector(descriptor, connector).await {
244+
Ok(stream) => return Ok(stream),
245+
Err(error) if tokio::time::Instant::now() >= deadline => return Err(error),
246+
Err(_) => tokio::time::sleep(Duration::from_millis(25)).await,
247+
}
220248
}
221249
}
250+
.await;
251+
attempts.span.record(
252+
"otel.status_code",
253+
if result.is_ok() { "OK" } else { "ERROR" },
254+
);
255+
result
222256
}
223257
}
224258

@@ -2334,7 +2368,9 @@ impl BoundaryClient {
23342368
confirm: Option<&RequestEnvelope>,
23352369
credential: &BoundaryCredential,
23362370
) -> Result<(tonic::transport::Channel, Arc<TransportAbort>), BackendError> {
2337-
let (channel, transport) = self.build_grpc_channel(credential).await?;
2371+
let (channel, transport) = self
2372+
.build_grpc_channel(credential, CONNECT_RETRY_TIMEOUT)
2373+
.await?;
23382374
let result = async {
23392375
self.exchange_on_channel(channel.clone(), attach, credential)
23402376
.await?;
@@ -2494,7 +2530,11 @@ impl BoundaryClient {
24942530
return Ok(cached.channel.clone());
24952531
}
24962532
let credential = BoundaryCredential::capture(&self.sandbox_bearer)?;
2497-
let (channel, transport) = self.build_grpc_channel(&credential).await?;
2533+
// Initial policy discovery and attach already have a startup deadline.
2534+
// Keep the transport wait in one connection span within that budget.
2535+
let (channel, transport) = self
2536+
.build_grpc_channel(&credential, self.attach_request_timeout)
2537+
.await?;
24982538
*state = Some(CachedGrpcChannel {
24992539
credential_epoch: credential.epoch,
25002540
bearer_fingerprint: credential.fingerprint,
@@ -2510,10 +2550,10 @@ impl BoundaryClient {
25102550
async fn build_grpc_channel(
25112551
&self,
25122552
credential: &BoundaryCredential,
2553+
connect_timeout: Duration,
25132554
) -> Result<(tonic::transport::Channel, Arc<TransportAbort>), BackendError> {
25142555
let runtime_descriptor = self.runtime_descriptor.clone();
25152556
let source = self.transport.clone();
2516-
let initial_connect_timeout = self.attach_request_timeout;
25172557
let transport = Arc::new(TransportAbort::default());
25182558
let connector_transport = transport.clone();
25192559
let authority = match &runtime_descriptor.transport {
@@ -2547,7 +2587,7 @@ impl BoundaryClient {
25472587
return Err(AbortableTransport::aborted_error());
25482588
}
25492589
source
2550-
.connect(&runtime_descriptor, initial_connect_timeout)
2590+
.connect(&runtime_descriptor, connect_timeout)
25512591
.await
25522592
.map(|inner| TokioIo::new(AbortableTransport { inner, abort }))
25532593
.map_err(|error| std::io::Error::other(error.to_string()))
@@ -3094,6 +3134,7 @@ pub mod test_support;
30943134

30953135
#[cfg(test)]
30963136
mod tests {
3137+
mod connection_tracing;
30973138
mod credential_renewal;
30983139
mod delegated;
30993140
mod flow_control;
@@ -4449,7 +4490,7 @@ mod tests {
44494490
}
44504491

44514492
#[tokio::test(start_paused = true)]
4452-
async fn configured_routed_bootstrap_wait_preserves_reconnect_window() {
4493+
async fn routed_connection_uses_the_callers_timeout() {
44534494
let certificate = test_certificate();
44544495
let mut descriptor =
44554496
tls_runtime_descriptor("127.0.0.1:1".parse().unwrap(), certificate.client_tls);
@@ -4461,7 +4502,7 @@ mod tests {
44614502
route: route.clone(),
44624503
};
44634504
let registration = router.register(&route, &descriptor.tls).unwrap();
4464-
let transport = BoundaryTransport::routed(&descriptor, registration.clone()).unwrap();
4505+
let transport = BoundaryTransport::routed(&descriptor, registration).unwrap();
44654506
let budget = Duration::from_mins(15);
44664507
let mut waiting = Box::pin(transport.connect(&descriptor, budget));
44674508
poll_fn(|context| {
@@ -4500,12 +4541,28 @@ mod tests {
45004541
.unwrap()
45014542
.unwrap();
45024543
drop(waiting);
4503-
assert!(registration.has_connected());
45044544
drop(stream);
45054545
drop(runtime.await.unwrap());
45064546

45074547
tokio::time::pause();
4508-
let mut reconnect = Box::pin(transport.connect(&descriptor, budget));
4548+
let mut attach = Box::pin(transport.connect(&descriptor, budget));
4549+
poll_fn(|context| {
4550+
assert!(attach.as_mut().poll(context).is_pending());
4551+
Poll::Ready(())
4552+
})
4553+
.await;
4554+
tokio::time::advance(CONNECT_RETRY_TIMEOUT + Duration::from_secs(1)).await;
4555+
poll_fn(|context| {
4556+
assert!(
4557+
attach.as_mut().poll(context).is_pending(),
4558+
"a previous connection must not shorten the startup budget for attach"
4559+
);
4560+
Poll::Ready(())
4561+
})
4562+
.await;
4563+
drop(attach);
4564+
4565+
let mut reconnect = Box::pin(transport.connect(&descriptor, CONNECT_RETRY_TIMEOUT));
45094566
poll_fn(|context| {
45104567
assert!(reconnect.as_mut().poll(context).is_pending());
45114568
Poll::Ready(())

0 commit comments

Comments
 (0)