diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 30c46e2fd96..6f8d0ffb3d4 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -34,6 +34,14 @@ pub use runtime::{ insert_mentions, migration, replica_fence, Db, DbConfig, DbPoolStats, DbReadinessOutcome, ReadSession, }; + +/// Valid low-cardinality `(pool_role, operation)` pairs for pool-acquisition telemetry. +pub const DB_POOL_ACQUIRE_VALID_PAIRS: [(&str, &str); 11] = + runtime::observability::POOL_ACQUIRE_VALID_PAIRS; + +/// Raw Prometheus series ceiling per relay pod for the operation-aware contract. +pub const DB_POOL_ACQUIRE_RAW_SERIES_PER_POD: usize = + runtime::observability::POOL_ACQUIRE_RAW_SERIES_PER_POD; pub(crate) use runtime::{ insert_mentions_in_transaction, observability, route_proof, ReadSessionInner, RouteDecision, RoutePredicate, diff --git a/crates/buzz-db/src/runtime/migration.rs b/crates/buzz-db/src/runtime/migration.rs index 68071e0ed24..f49d73ae812 100644 --- a/crates/buzz-db/src/runtime/migration.rs +++ b/crates/buzz-db/src/runtime/migration.rs @@ -82,9 +82,12 @@ where F: FnOnce(PgConnection) -> Fut, Fut: Future)>, { - let mut lock_conn = crate::observability::acquire(pool, crate::observability::PoolRole::Writer) - .await? - .detach(); + let mut lock_conn = crate::observability::acquire_writer_with_legacy_metrics( + pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await? + .detach(); // This dedicated connection intentionally waits for the current migration // or schema-destruction owner and may then run long DDL. Exempt those two // phases from runtime lock/statement budgets. Keep the idle-in-transaction diff --git a/crates/buzz-db/src/runtime/mod.rs b/crates/buzz-db/src/runtime/mod.rs index 214cc4bca60..5f608cb78fc 100644 --- a/crates/buzz-db/src/runtime/mod.rs +++ b/crates/buzz-db/src/runtime/mod.rs @@ -24,7 +24,9 @@ pub async fn insert_mentions( event: &nostr::Event, channel_id: Option, ) -> Result<()> { - let mut tx = pool.begin().await?; + let connection = + observability::acquire_writer(pool, observability::WriterOperation::EventWrite).await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; insert_mentions_in_transaction(&mut tx, community_id, event, channel_id).await?; tx.commit().await?; Ok(()) @@ -687,25 +689,43 @@ impl Db { return; }; let aurora_identity = self.reader_aurora_identity.clone(); - tokio::spawn(async move { - match observability::acquire(&read_pool, observability::PoolRole::Reader).await { - Ok(mut conn) => { - tracing::info!("read replica reachable at boot"); - match replica_fence::reader_supports_aurora_identity(&mut conn).await { - Ok(supported) => { - let _ = aurora_identity.set(supported); - } - Err(e) => tracing::debug!( - error = %e, - "aurora identity boot prime failed; first routed read will probe" - ), + tokio::spawn(Self::read_pool_boot_ping_once(read_pool, aurora_identity)); + } + + async fn read_pool_boot_ping_once( + read_pool: PgPool, + aurora_identity: std::sync::Arc>, + ) { + match observability::acquire_reader_with_legacy_metrics( + &read_pool, + observability::ReaderOperation::Bootstrap, + ) + .await + { + Ok(mut conn) => { + tracing::info!("read replica reachable at boot"); + match replica_fence::reader_supports_aurora_identity(&mut conn).await { + Ok(supported) => { + let _ = aurora_identity.set(supported); } + Err(e) => tracing::debug!( + error = %e, + "aurora identity boot prime failed; first routed read will probe" + ), } - Err(e) => tracing::warn!( - "read replica unreachable at boot; serving all-writer until it recovers: {e}" - ), } - }); + Err(e) => tracing::warn!( + "read replica unreachable at boot; serving all-writer until it recovers: {e}" + ), + } + } + + #[cfg(test)] + pub(crate) async fn read_pool_boot_ping_for_tests(&self) { + let Some(read_pool) = self.read_pool.clone() else { + return; + }; + Self::read_pool_boot_ping_once(read_pool, self.reader_aurora_identity.clone()).await; } /// Creates a `Db` from an existing `PgPool` (useful in tests). @@ -771,8 +791,7 @@ impl Db { if self.read_pool.is_none() { return Ok(false); } - replica_fence::verify_floor_guard_catalog(&self.pool).await?; - replica_fence::verify_floor_guard_behavior(&self.pool).await?; + self.verify_replica_fence_at_boot().await?; tokio::spawn(replica_fence::run_probe( self.pool.clone(), std::sync::Arc::clone(&self.fence), @@ -780,6 +799,17 @@ impl Db { Ok(true) } + /// Verify replica-fence catalog shape and behavior through attributed + /// writer/bootstrap acquisitions without starting the recurring probe. + pub(crate) async fn verify_replica_fence_at_boot(&self) -> Result<()> { + let mut connection = + observability::acquire_writer(&self.pool, observability::WriterOperation::Bootstrap) + .await?; + replica_fence::verify_floor_guard_catalog(&mut *connection).await?; + drop(connection); + replica_fence::verify_floor_guard_behavior(&self.pool).await + } + /// The pool for lag-tolerant reads: the read replica when configured, /// otherwise the writer pool. /// @@ -817,6 +847,7 @@ impl Db { async fn proved_reader( &self, read_pool: &PgPool, + operation: observability::ReaderOperation, ) -> std::result::Result< ( sqlx::Transaction<'static, sqlx::Postgres>, @@ -830,7 +861,9 @@ impl Db { // `read_pool` separately would spend a second budget whenever the // capability is uncached — i.e. after a failed boot ping, which is // precisely the reader-unavailable case the bound must hold for. - let conn = match observability::acquire(read_pool, observability::PoolRole::Reader).await { + let conn = match observability::acquire_reader_with_legacy_metrics(read_pool, operation) + .await + { Ok(conn) => conn, Err(sqlx::Error::PoolTimedOut) => { tracing::warn!("reader pool acquire timed out; routing to writer"); @@ -953,7 +986,16 @@ impl Db { /// Returns `true` if the database is reachable. pub async fn ping(&self) -> bool { - sqlx::query("SELECT 1").execute(&self.pool).await.is_ok() + let Ok(mut connection) = + observability::acquire_writer(&self.pool, observability::WriterOperation::Readiness) + .await + else { + return false; + }; + sqlx::query("SELECT 1") + .execute(&mut *connection) + .await + .is_ok() } /// Checks writer-pool acquisition and query execution against one deadline. @@ -974,14 +1016,19 @@ impl Db { deadline: tokio::time::Instant, query: &'static str, ) -> DbReadinessOutcome { - let mut connection = match tokio::time::timeout_at(deadline, self.pool.acquire()).await { - Err(_) => return DbReadinessOutcome::PoolTimeout, - Ok(Err(sqlx::Error::PoolTimedOut)) => return DbReadinessOutcome::PoolTimeout, - Ok(Err(error)) => { + let mut connection = match observability::acquire_writer_until( + &self.pool, + observability::WriterOperation::Readiness, + deadline, + ) + .await + { + Err(sqlx::Error::PoolTimedOut) => return DbReadinessOutcome::PoolTimeout, + Err(error) => { tracing::debug!(error = %error, "Postgres readiness pool acquisition failed"); return DbReadinessOutcome::PoolError; } - Ok(Ok(connection)) => connection, + Ok(connection) => connection, }; match tokio::time::timeout_at(deadline, sqlx::query(query).execute(&mut *connection)).await @@ -1008,6 +1055,15 @@ impl Db { } } + /// Refresh all expected operation-specific waiter gauges, including zero. + /// + /// The relay pool sampler calls this periodically so an exporter idle + /// timeout cannot make a healthy zero indistinguishable from missing + /// telemetry. + pub fn refresh_pool_waiter_metrics(&self) { + observability::refresh_pool_waiters(self.read_pool.is_some()); + } + /// Pool utilisation stats for the read-replica pool, when configured. /// /// `max` is the **reader's** ceiling ([`Db::read_max_connections`]), not @@ -1028,14 +1084,29 @@ impl Db { /// /// Returns a `'static` transaction because `PgPool` is `Arc`-backed internally. /// The transaction holds an owned pool handle, not a borrow. - pub async fn begin_transaction(&self) -> Result> { - let connection = - observability::acquire(&self.pool, observability::PoolRole::Writer).await?; + pub async fn begin_event_write_transaction( + &self, + ) -> Result> { + let connection = observability::acquire_writer_with_legacy_metrics( + &self.pool, + observability::WriterOperation::EventWrite, + ) + .await?; sqlx::Transaction::begin(connection, None) .await .map_err(Into::into) } + /// Begin an event-write transaction through the pre-operation API name. + /// + /// New callers should use [`Self::begin_event_write_transaction`] so the + /// semantic intent is explicit. This alias preserves the crate's public + /// API while emitting the same operation-aware and compatibility metrics. + #[deprecated(note = "use Db::begin_event_write_transaction")] + pub async fn begin_transaction(&self) -> Result> { + self.begin_event_write_transaction().await + } + /// Insert an event while holding and validating an admitted serving-write /// lease under the community ordering lock through commit. /// @@ -1058,7 +1129,10 @@ impl Db { return Err(DbError::EphemeralEventRejected(kind_u16)); } - let mut tx = self.pool.begin().await?; + let connection = + observability::acquire_writer(&self.pool, observability::WriterOperation::EventWrite) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; self.deletion_store() .guard_transaction_with_serving_lease(&mut tx, lease) .await?; @@ -1086,6 +1160,7 @@ impl Db { &self, path: &'static str, predicate: RoutePredicate, + operation: observability::ReaderOperation, ) -> RouteDecision { let Some(read_pool) = &self.read_pool else { Self::record_route(path, "writer", "disabled"); @@ -1128,7 +1203,7 @@ impl Db { Self::record_route(path, "writer", reason); return RouteDecision::Writer; } - match self.proved_reader(read_pool).await { + match self.proved_reader(read_pool, operation).await { Ok((tx, entry)) => { // Re-evaluate against the entry the session actually proved // (it may be older than the shared newest). diff --git a/crates/buzz-db/src/runtime/observability.rs b/crates/buzz-db/src/runtime/observability.rs index 0d4774ccad6..4d2e5f7ad69 100644 --- a/crates/buzz-db/src/runtime/observability.rs +++ b/crates/buzz-db/src/runtime/observability.rs @@ -4,26 +4,156 @@ //! never derive labels from tenant data, events, SQL text, or query identifiers. use std::future::Future; +use std::sync::Mutex; use std::time::{Duration, Instant}; +/// One valid pool/operation acquisition family. +/// +/// Keeping role and operation in one enum makes invalid combinations +/// unrepresentable at call sites and gives the series budget one exhaustive +/// source of truth. +#[repr(usize)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) enum PoolRole { - Writer, - Reader, +enum PoolOperation { + WriterBootstrap, + ReaderBootstrap, + WriterReadiness, + WriterTenantResolution, + WriterAuthentication, + WriterAuthorization, + ReaderAuthorization, + WriterSubscriptionHistory, + ReaderSubscriptionHistory, + WriterEventWrite, + WriterMaintenance, } -impl PoolRole { +/// Writer-pool operations. Reader-only combinations cannot be constructed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum WriterOperation { + Bootstrap, + Readiness, + TenantResolution, + Authentication, + Authorization, + SubscriptionHistory, + EventWrite, + Maintenance, +} + +impl WriterOperation { #[cfg(test)] - pub(crate) const ALL: [Self; 2] = [Self::Writer, Self::Reader]; + const ALL: [Self; 8] = [ + Self::Bootstrap, + Self::Readiness, + Self::TenantResolution, + Self::Authentication, + Self::Authorization, + Self::SubscriptionHistory, + Self::EventWrite, + Self::Maintenance, + ]; - pub(crate) const fn as_str(self) -> &'static str { + const fn pair(self) -> PoolOperation { + match self { + Self::Bootstrap => PoolOperation::WriterBootstrap, + Self::Readiness => PoolOperation::WriterReadiness, + Self::TenantResolution => PoolOperation::WriterTenantResolution, + Self::Authentication => PoolOperation::WriterAuthentication, + Self::Authorization => PoolOperation::WriterAuthorization, + Self::SubscriptionHistory => PoolOperation::WriterSubscriptionHistory, + Self::EventWrite => PoolOperation::WriterEventWrite, + Self::Maintenance => PoolOperation::WriterMaintenance, + } + } +} + +/// Reader-pool operations. Writer-only combinations cannot be constructed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReaderOperation { + Bootstrap, + Authorization, + SubscriptionHistory, +} + +impl ReaderOperation { + #[cfg(test)] + const ALL: [Self; 3] = [ + Self::Bootstrap, + Self::Authorization, + Self::SubscriptionHistory, + ]; + + const fn pair(self) -> PoolOperation { + match self { + Self::Bootstrap => PoolOperation::ReaderBootstrap, + Self::Authorization => PoolOperation::ReaderAuthorization, + Self::SubscriptionHistory => PoolOperation::ReaderSubscriptionHistory, + } + } +} + +impl PoolOperation { + pub(crate) const ALL: [Self; 11] = [ + Self::WriterBootstrap, + Self::ReaderBootstrap, + Self::WriterReadiness, + Self::WriterTenantResolution, + Self::WriterAuthentication, + Self::WriterAuthorization, + Self::ReaderAuthorization, + Self::WriterSubscriptionHistory, + Self::ReaderSubscriptionHistory, + Self::WriterEventWrite, + Self::WriterMaintenance, + ]; + + pub(crate) const fn pool_role(self) -> &'static str { + match self { + Self::ReaderBootstrap | Self::ReaderAuthorization | Self::ReaderSubscriptionHistory => { + "reader" + } + _ => "writer", + } + } + + pub(crate) const fn operation(self) -> &'static str { match self { - Self::Writer => "writer", - Self::Reader => "reader", + Self::WriterBootstrap | Self::ReaderBootstrap => "bootstrap", + Self::WriterReadiness => "readiness", + Self::WriterTenantResolution => "tenant_resolution", + Self::WriterAuthentication => "authentication", + Self::WriterAuthorization | Self::ReaderAuthorization => "authorization", + Self::WriterSubscriptionHistory | Self::ReaderSubscriptionHistory => { + "subscription_history" + } + Self::WriterEventWrite => "event_write", + Self::WriterMaintenance => "maintenance", } } + + const fn index(self) -> usize { + self as usize + } } +pub(crate) const POOL_ACQUIRE_VALID_PAIRS: [(&str, &str); 11] = [ + ("writer", "bootstrap"), + ("reader", "bootstrap"), + ("writer", "readiness"), + ("writer", "tenant_resolution"), + ("writer", "authentication"), + ("writer", "authorization"), + ("reader", "authorization"), + ("writer", "subscription_history"), + ("reader", "subscription_history"), + ("writer", "event_write"), + ("writer", "maintenance"), +]; + +/// Eleven valid pairs × (12 histogram series + 4 outcome counters + 1 gauge). +pub(crate) const POOL_ACQUIRE_RAW_SERIES_PER_POD: usize = POOL_ACQUIRE_VALID_PAIRS.len() * 17; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum LockType { Replacement, @@ -59,17 +189,19 @@ pub(crate) enum Outcome { Success, Error, Timeout, + Cancelled, } impl Outcome { #[cfg(test)] - pub(crate) const ALL: [Self; 3] = [Self::Success, Self::Error, Self::Timeout]; + pub(crate) const ALL: [Self; 4] = [Self::Success, Self::Error, Self::Timeout, Self::Cancelled]; pub(crate) const fn as_str(self) -> &'static str { match self { Self::Success => "success", Self::Error => "error", Self::Timeout => "timeout", + Self::Cancelled => "cancelled", } } @@ -115,42 +247,256 @@ impl TransactionOperation { Self::FenceCommunityDeletion => "fence_community_deletion", } } + + const fn writer_operation(self) -> WriterOperation { + match self { + Self::ReplaceParameterizedEvent + | Self::ReplaceAddressableEvent + | Self::PublishNip43MembershipLocked + | Self::AcceptPushLeaseEvent => WriterOperation::EventWrite, + Self::BeginCommunityDeletionQuiescing | Self::FenceCommunityDeletion => { + WriterOperation::Maintenance + } + } + } } -pub(crate) fn record_pool_acquire(role: PoolRole, outcome: Outcome, elapsed: Duration) { +fn record_pool_acquire( + pair: PoolOperation, + outcome: Outcome, + elapsed: Duration, + emit_legacy: bool, +) { + // Preserve the original observed population for existing dashboards. + // Newly instrumented raw-pool seams must not create a deployment-time + // discontinuity in these compatibility families. + if emit_legacy && outcome != Outcome::Cancelled { + metrics::histogram!( + "buzz_db_pool_acquire_wait_seconds", + "pool_role" => pair.pool_role(), + "outcome" => outcome.as_str(), + ) + .record(elapsed.as_secs_f64()); + metrics::counter!( + "buzz_db_pool_acquisitions_total", + "pool_role" => pair.pool_role(), + "outcome" => outcome.as_str(), + ) + .increment(1); + } + metrics::histogram!( - "buzz_db_pool_acquire_wait_seconds", - "pool_role" => role.as_str(), - "outcome" => outcome.as_str(), + "buzz_db_pool_acquire_duration_seconds", + "pool_role" => pair.pool_role(), + "operation" => pair.operation(), ) .record(elapsed.as_secs_f64()); metrics::counter!( - "buzz_db_pool_acquisitions_total", - "pool_role" => role.as_str(), + "buzz_db_pool_acquire_attempts_total", + "pool_role" => pair.pool_role(), + "operation" => pair.operation(), "outcome" => outcome.as_str(), ) .increment(1); } -pub(crate) async fn acquire( +static POOL_WAITERS: [Mutex; PoolOperation::ALL.len()] = + [const { Mutex::new(0) }; PoolOperation::ALL.len()]; + +#[cfg(test)] +static POOL_METRICS_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +#[cfg(test)] +#[derive(Clone)] +struct WaiterPublishTestHook { + pair: PoolOperation, + value: u64, + entered: std::sync::Arc, + release: std::sync::Arc, + armed: std::sync::Arc, +} + +#[cfg(test)] +static WAITER_PUBLISH_TEST_HOOK: Mutex> = Mutex::new(None); + +#[cfg(test)] +static WAITER_LAST_PUBLISHED: [Mutex; PoolOperation::ALL.len()] = + [const { Mutex::new(u64::MAX) }; PoolOperation::ALL.len()]; + +fn publish_waiters(pair: PoolOperation, value: u64) { + #[cfg(test)] + { + let hook = WAITER_PUBLISH_TEST_HOOK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if let Some(hook) = hook { + if hook.pair == pair + && hook.value == value + && hook.armed.swap(false, std::sync::atomic::Ordering::SeqCst) + { + hook.entered.wait(); + hook.release.wait(); + } + } + *WAITER_LAST_PUBLISHED[pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = value; + } + metrics::gauge!( + "buzz_db_pool_waiters", + "pool_role" => pair.pool_role(), + "operation" => pair.operation(), + ) + .set(value as f64); +} + +/// Re-publish every valid waiter pair, including healthy zero, so exporter +/// idle eviction cannot turn an expected zero into ambiguous missing data. +pub(crate) fn refresh_pool_waiters(include_reader: bool) { + for pair in PoolOperation::ALL { + if pair.pool_role() == "reader" && !include_reader { + continue; + } + let waiters = POOL_WAITERS[pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + publish_waiters(pair, *waiters); + } +} + +/// Owns one polled connection acquisition until exactly one terminal. +/// +/// Because async function bodies do not run until first poll, a future that is +/// constructed and immediately dropped emits nothing. Once armed, dropping it +/// while awaiting SQLx records `cancelled`, duration, and the balanced waiter +/// decrement. +struct PoolAcquireAttempt { + pair: PoolOperation, + started: Instant, + emit_legacy: bool, + terminal: bool, +} + +impl PoolAcquireAttempt { + fn start(pair: PoolOperation, emit_legacy: bool) -> Self { + { + let mut waiters = POOL_WAITERS[pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *waiters += 1; + publish_waiters(pair, *waiters); + } + Self { + pair, + started: Instant::now(), + emit_legacy, + terminal: false, + } + } + + fn finish(mut self, outcome: Outcome) { + self.terminal = true; + record_pool_acquire(self.pair, outcome, self.started.elapsed(), self.emit_legacy); + self.release_waiter(); + } + + fn release_waiter(&self) { + let mut waiters = POOL_WAITERS[self.pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + debug_assert!(*waiters > 0, "pool waiter balance underflow"); + *waiters = waiters.saturating_sub(1); + publish_waiters(self.pair, *waiters); + } +} + +impl Drop for PoolAcquireAttempt { + fn drop(&mut self) { + if !self.terminal { + record_pool_acquire( + self.pair, + Outcome::Cancelled, + self.started.elapsed(), + self.emit_legacy, + ); + self.release_waiter(); + self.terminal = true; + } + } +} + +async fn acquire( pool: &sqlx::PgPool, - role: PoolRole, + pair: PoolOperation, + emit_legacy: bool, ) -> sqlx::Result> { - let started = Instant::now(); + let attempt = PoolAcquireAttempt::start(pair, emit_legacy); let result = pool.acquire().await; let outcome = result .as_ref() .map(|_| Outcome::Success) .unwrap_or_else(Outcome::from_sqlx_error); - record_pool_acquire(role, outcome, started.elapsed()); + attempt.finish(outcome); result } +/// Acquire from an authoritative writer pool for one valid writer operation. +pub(crate) async fn acquire_writer( + pool: &sqlx::PgPool, + operation: WriterOperation, +) -> sqlx::Result> { + acquire(pool, operation.pair(), false).await +} + +/// Acquire from a writer seam already covered by the pre-operation metric. +pub(crate) async fn acquire_writer_with_legacy_metrics( + pool: &sqlx::PgPool, + operation: WriterOperation, +) -> sqlx::Result> { + acquire(pool, operation.pair(), true).await +} + +/// Acquire from a reader seam already covered by the pre-operation metric. +pub(super) async fn acquire_reader_with_legacy_metrics( + pool: &sqlx::PgPool, + operation: ReaderOperation, +) -> sqlx::Result> { + acquire(pool, operation.pair(), true).await +} + +/// Acquire within an operation-owned absolute deadline. +/// +/// A deadline expiry is a timeout terminal. Dropping the enclosing future +/// before that deadline remains a cancellation terminal. +pub(crate) async fn acquire_writer_until( + pool: &sqlx::PgPool, + operation: WriterOperation, + deadline: tokio::time::Instant, +) -> sqlx::Result> { + let pair = operation.pair(); + let attempt = PoolAcquireAttempt::start(pair, false); + match tokio::time::timeout_at(deadline, pool.acquire()).await { + Err(_) => { + attempt.finish(Outcome::Timeout); + Err(sqlx::Error::PoolTimedOut) + } + Ok(result) => { + let outcome = result + .as_ref() + .map(|_| Outcome::Success) + .unwrap_or_else(Outcome::from_sqlx_error); + attempt.finish(outcome); + result + } + } +} + pub(crate) async fn begin_transaction( pool: &sqlx::PgPool, operation: TransactionOperation, ) -> sqlx::Result<(sqlx::Transaction<'static, sqlx::Postgres>, TransactionTimer)> { - let connection = acquire(pool, PoolRole::Writer).await?; + let connection = acquire_writer_with_legacy_metrics(pool, operation.writer_operation()).await?; let transaction = sqlx::Transaction::begin(connection, None).await?; Ok((transaction, TransactionTimer::start(operation))) } @@ -221,16 +567,44 @@ impl Drop for TransactionTimer { #[cfg(test)] mod tests { use super::{ - acquire, observe_advisory_lock, record_pool_acquire, LockType, Outcome, PoolRole, - TransactionOperation, TransactionTimer, + acquire_reader_with_legacy_metrics, acquire_writer, acquire_writer_with_legacy_metrics, + observe_advisory_lock, record_pool_acquire, refresh_pool_waiters, LockType, Outcome, + PoolAcquireAttempt, PoolOperation, ReaderOperation, TransactionOperation, TransactionTimer, + WriterOperation, }; use metrics_util::debugging::{DebugValue, DebuggingRecorder}; use std::collections::{BTreeMap, BTreeSet}; + use std::sync::{Arc, Barrier}; use std::time::Duration; #[test] fn label_vocabularies_are_closed_and_documented() { - assert_eq!(PoolRole::ALL.map(PoolRole::as_str), ["writer", "reader"]); + assert_eq!( + PoolOperation::ALL.map(|pair| (pair.pool_role(), pair.operation())), + super::POOL_ACQUIRE_VALID_PAIRS + ); + assert_eq!( + WriterOperation::ALL.map(WriterOperation::pair), + [ + PoolOperation::WriterBootstrap, + PoolOperation::WriterReadiness, + PoolOperation::WriterTenantResolution, + PoolOperation::WriterAuthentication, + PoolOperation::WriterAuthorization, + PoolOperation::WriterSubscriptionHistory, + PoolOperation::WriterEventWrite, + PoolOperation::WriterMaintenance, + ] + ); + assert_eq!( + ReaderOperation::ALL.map(ReaderOperation::pair), + [ + PoolOperation::ReaderBootstrap, + PoolOperation::ReaderAuthorization, + PoolOperation::ReaderSubscriptionHistory, + ] + ); + assert_eq!(super::POOL_ACQUIRE_RAW_SERIES_PER_POD, 187); assert_eq!( LockType::ALL.map(LockType::as_str), [ @@ -243,7 +617,7 @@ mod tests { ); assert_eq!( Outcome::ALL.map(Outcome::as_str), - ["success", "error", "timeout"] + ["success", "error", "timeout", "cancelled"] ); assert_eq!( TransactionOperation::ALL.map(TransactionOperation::as_str), @@ -311,14 +685,16 @@ mod tests { let _guard = metrics::set_default_local_recorder(&recorder); record_pool_acquire( - PoolRole::Writer, + PoolOperation::WriterReadiness, Outcome::Success, Duration::from_millis(12), + false, ); record_pool_acquire( - PoolRole::Reader, + PoolOperation::ReaderSubscriptionHistory, Outcome::Timeout, Duration::from_millis(34), + true, ); let lock_ok: sqlx::Result<()> = observe_advisory_lock(LockType::Replacement, async { Ok(()) }).await; @@ -353,18 +729,10 @@ mod tests { .collect::>(); for expected in [ - ( - "buzz_db_pool_acquire_wait_seconds", - [("outcome", "success"), ("pool_role", "writer")], - ), ( "buzz_db_pool_acquire_wait_seconds", [("outcome", "timeout"), ("pool_role", "reader")], ), - ( - "buzz_db_pool_acquisitions_total", - [("outcome", "success"), ("pool_role", "writer")], - ), ( "buzz_db_pool_acquisitions_total", [("outcome", "timeout"), ("pool_role", "reader")], @@ -410,6 +778,65 @@ mod tests { "missing metric series {expected:?}; got {keys:?}" ); } + for name in [ + "buzz_db_pool_acquire_wait_seconds", + "buzz_db_pool_acquisitions_total", + ] { + assert!( + !keys.contains(&( + name.to_owned(), + [ + ("outcome".to_owned(), "success".to_owned()), + ("pool_role".to_owned(), "writer".to_owned()), + ] + .into_iter() + .collect(), + )), + "newly instrumented seams must not expand legacy metric population" + ); + } + + for (name, labels) in [ + ( + "buzz_db_pool_acquire_duration_seconds", + [("operation", "readiness"), ("pool_role", "writer")], + ), + ( + "buzz_db_pool_acquire_duration_seconds", + [ + ("operation", "subscription_history"), + ("pool_role", "reader"), + ], + ), + ] { + let labels = labels + .into_iter() + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect(); + assert!( + keys.contains(&(name.to_owned(), labels)), + "missing operation-aware pool duration for {name}" + ); + } + for (pool_role, operation, outcome) in [ + ("writer", "readiness", "success"), + ("reader", "subscription_history", "timeout"), + ] { + assert!(keys.contains(&( + "buzz_db_pool_acquire_attempts_total".to_owned(), + [ + ("operation".to_owned(), operation.to_owned()), + ("outcome".to_owned(), outcome.to_owned()), + ("pool_role".to_owned(), pool_role.to_owned()), + ] + .into_iter() + .collect(), + ))); + } + assert!(keys.iter().all(|(name, labels)| { + name != "buzz_db_pool_acquire_duration_seconds" + || (!labels.contains_key("outcome") && !labels.contains_key("result")) + })); for (key, _, _, value) in snapshot { if key.key().name().ends_with("_seconds") { @@ -426,44 +853,391 @@ mod tests { } } + #[test] + fn cancelled_attempt_records_terminal_and_refreshes_zero() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.blocking_lock(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + let attempt = PoolAcquireAttempt::start(PoolOperation::WriterTenantResolution, false); + drop(attempt); + refresh_pool_waiters(true); + + let mut saw_cancelled = false; + let mut saw_zero = false; + for (key, _, _, value) in snapshotter.snapshot().into_vec() { + let labels = key + .key() + .labels() + .map(|label| (label.key(), label.value())) + .collect::>(); + if labels.get("operation") != Some(&"tenant_resolution") { + continue; + } + match key.key().name() { + "buzz_db_pool_acquire_attempts_total" => { + let DebugValue::Counter(value) = value else { + panic!("attempt terminals must be a counter"); + }; + saw_cancelled = labels.get("outcome") == Some(&"cancelled") && value == 1; + } + "buzz_db_pool_waiters" => { + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be a gauge"); + }; + saw_zero = value.into_inner() == 0.0; + } + _ => {} + } + } + assert!( + saw_cancelled, + "dropped armed attempt must terminalize cancellation" + ); + assert!(saw_zero, "periodic refresh must publish a healthy zero"); + } + + #[test] + fn waiter_refresh_omits_reader_pairs_when_no_reader_pool_is_configured() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.blocking_lock(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + refresh_pool_waiters(false); + + let published = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_waiters" { + return None; + } + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be gauges"); + }; + assert_eq!(value.into_inner(), 0.0); + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + Some((labels["pool_role"].clone(), labels["operation"].clone())) + }) + .collect::>(); + let expected = WriterOperation::ALL + .into_iter() + .map(|operation| { + let pair = operation.pair(); + (pair.pool_role().to_owned(), pair.operation().to_owned()) + }) + .collect::>(); + + assert_eq!(published, expected); + assert!(published.iter().all(|(pool_role, _)| pool_role == "writer")); + } + + #[tokio::test(flavor = "current_thread")] + async fn compatibility_metrics_only_cover_preexisting_acquisition_seams() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.lock().await; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy(&crate::test_support::database_url()) + .expect("construct lazy compatibility test pool"); + pool.close().await; + + let error = acquire_writer(&pool, WriterOperation::EventWrite) + .await + .expect_err("closed newly instrumented seam errors"); + assert!(matches!(error, sqlx::Error::PoolClosed)); + assert_eq!( + legacy_acquisition_count(&snapshotter.snapshot().into_vec()), + 0 + ); + + let error = acquire_writer_with_legacy_metrics(&pool, WriterOperation::EventWrite) + .await + .expect_err("closed legacy seam errors"); + assert!(matches!(error, sqlx::Error::PoolClosed)); + assert_eq!( + legacy_acquisition_count(&snapshotter.snapshot().into_vec()), + 1 + ); + } + + fn legacy_acquisition_count( + snapshot: &[( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, + )], + ) -> u64 { + snapshot + .iter() + .filter_map(|(key, _, _, value)| { + (key.key().name() == "buzz_db_pool_acquisitions_total") + .then_some(value) + .map(|value| match value { + DebugValue::Counter(value) => *value, + _ => panic!("legacy acquisitions must be a counter"), + }) + }) + .sum() + } + + #[test] + fn concurrent_attempts_publish_an_exact_balanced_waiter_count() { + const ATTEMPTS: usize = 8; + + let _test_guard = super::POOL_METRICS_TEST_LOCK.blocking_lock(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let armed = Arc::new(Barrier::new(ATTEMPTS + 1)); + let release = Arc::new(Barrier::new(ATTEMPTS + 1)); + let threads = (0..ATTEMPTS) + .map(|_| { + let armed = Arc::clone(&armed); + let release = Arc::clone(&release); + std::thread::spawn(move || { + let attempt = + PoolAcquireAttempt::start(PoolOperation::WriterTenantResolution, false); + armed.wait(); + release.wait(); + drop(attempt); + }) + }) + .collect::>(); + + armed.wait(); + refresh_pool_waiters(true); + let live = waiter_value( + &snapshotter.snapshot().into_vec(), + "writer", + "tenant_resolution", + ); + assert_eq!(live, Some(ATTEMPTS as f64)); + + release.wait(); + for thread in threads { + thread.join().expect("waiter thread completes"); + } + refresh_pool_waiters(true); + let balanced = waiter_value( + &snapshotter.snapshot().into_vec(), + "writer", + "tenant_resolution", + ); + assert_eq!(balanced, Some(0.0)); + } + + #[test] + fn waiter_publication_is_serialized_with_state_mutation() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.blocking_lock(); + let pair = PoolOperation::WriterTenantResolution; + let first = PoolAcquireAttempt::start(pair, false); + let second = PoolAcquireAttempt::start(pair, false); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + *super::WAITER_PUBLISH_TEST_HOOK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = + Some(super::WaiterPublishTestHook { + pair, + value: 1, + entered: Arc::clone(&entered), + release: Arc::clone(&release), + armed: Arc::new(std::sync::atomic::AtomicBool::new(true)), + }); + + let first_drop = std::thread::spawn(move || drop(first)); + entered.wait(); + let mutation_lock_held = super::POOL_WAITERS[pair.index()].try_lock().is_err(); + release.wait(); + first_drop.join().expect("first drop completes"); + drop(second); + *super::WAITER_PUBLISH_TEST_HOOK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = None; + + assert!( + mutation_lock_held, + "waiter state mutation must remain locked until its publication completes" + ); + assert_eq!( + *super::WAITER_LAST_PUBLISHED[pair.index()] + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + 0, + "the final directly published waiter value must be balanced without a refresh" + ); + } + + fn waiter_value( + snapshot: &[( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, + )], + pool_role: &str, + operation: &str, + ) -> Option { + snapshot.iter().find_map(|(key, _, _, value)| { + let labels = key.key().labels().collect::>(); + if key.key().name() != "buzz_db_pool_waiters" + || !labels + .iter() + .any(|label| label.key() == "pool_role" && label.value() == pool_role) + || !labels + .iter() + .any(|label| label.key() == "operation" && label.value() == operation) + { + return None; + } + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be gauges"); + }; + Some(value.into_inner()) + }) + } + async fn pool_acquire_records_success_timeout_and_error_with_wait_time() { // This timeout also bounds the pool's initial connection. Leave enough // headroom for a cold PostgreSQL start under the lane's eight workers; - // the assertion below cares about classification, not a 75 ms budget. - let database_url = std::env::var("TEST_DATABASE_URL") - .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_owned()); // sadscan:disable np.postgres.1 -- local test-only credentials + // the assertion below cares about classification, not a sub-second + // synthetic timeout budget. + let database_url = crate::test_support::database_url(); let pool = sqlx::postgres::PgPoolOptions::new() .max_connections(1) - .acquire_timeout(Duration::from_secs(1)) + .acquire_timeout(Duration::from_secs(5)) .connect(&database_url) .await .expect("connect size-one test pool"); + let reader_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(5)) + .connect(&database_url) + .await + .expect("connect size-one reader test pool"); let recorder = DebuggingRecorder::new(); let snapshotter = recorder.snapshotter(); let _guard = metrics::set_default_local_recorder(&recorder); - let held = acquire(&pool, PoolRole::Writer) + let held = acquire_writer_with_legacy_metrics(&pool, WriterOperation::EventWrite) .await .expect("writer acquire succeeds"); - let timeout = acquire(&pool, PoolRole::Reader) + let mut cancelled = Box::pin(acquire_writer_with_legacy_metrics( + &pool, + WriterOperation::Authentication, + )); + tokio::select! { + result = &mut cancelled => panic!("blocked acquisition unexpectedly completed: {result:?}"), + () = tokio::time::sleep(Duration::from_millis(40)) => {} + } + let before_cancel = snapshotter.snapshot().into_vec(); + let live_waiter = before_cancel.iter().find_map(|(key, _, _, value)| { + let labels = key.key().labels().collect::>(); + if key.key().name() != "buzz_db_pool_waiters" + || !labels + .iter() + .any(|label| label.key() == "pool_role" && label.value() == "writer") + || !labels + .iter() + .any(|label| label.key() == "operation" && label.value() == "authentication") + { + return None; + } + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be a gauge"); + }; + Some(value.into_inner()) + }); + assert_eq!(live_waiter, Some(1.0)); + let legacy_before_cancel = legacy_acquisition_count(&before_cancel); + assert_eq!( + legacy_before_cancel, 1, + "the completed legacy acquisition must be counted exactly once" + ); + let writer_success = before_cancel.iter().any(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_acquire_wait_seconds" { + return false; + } + let labels = key.key().labels().collect::>(); + let DebugValue::Histogram(samples) = value else { + panic!("pool wait must be a histogram"); + }; + labels + .iter() + .any(|label| label.key() == "pool_role" && label.value() == "writer") + && labels + .iter() + .any(|label| label.key() == "outcome" && label.value() == "success") + && !samples.is_empty() + }); + drop(cancelled); + let after_cancel = snapshotter.snapshot().into_vec(); + let legacy_after_cancel = legacy_acquisition_count(&after_cancel); + let mut cancelled_terminal = None; + let mut balanced_waiter = None; + for (key, _, _, value) in after_cancel { + let labels = key.key().labels().collect::>(); + let label = |name: &str| { + labels + .iter() + .find(|label| label.key() == name) + .map(|label| label.value()) + }; + if label("pool_role") != Some("writer") || label("operation") != Some("authentication") + { + continue; + } + match key.key().name() { + "buzz_db_pool_waiters" => { + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be a gauge"); + }; + balanced_waiter = Some(value.into_inner()); + } + "buzz_db_pool_acquire_attempts_total" if label("outcome") == Some("cancelled") => { + let DebugValue::Counter(value) = value else { + panic!("cancelled acquisition terminal must be a counter"); + }; + cancelled_terminal = Some(value); + } + _ => {} + } + } + assert_eq!(balanced_waiter, Some(0.0)); + assert_eq!(cancelled_terminal, Some(1)); + assert_eq!( + legacy_after_cancel, 0, + "cancelling a legacy seam must not expand its historical population" + ); + let held_reader = reader_pool + .acquire() .await - .expect_err("reader-labeled checkout times out while pool is saturated"); + .expect("hold the reader test connection"); + let timeout = + acquire_reader_with_legacy_metrics(&reader_pool, ReaderOperation::SubscriptionHistory) + .await + .expect_err("reader-labeled checkout times out while pool is saturated"); assert!(matches!(timeout, sqlx::Error::PoolTimedOut)); + drop(held_reader); drop(held); pool.close().await; - let closed = acquire(&pool, PoolRole::Writer) + let closed = acquire_writer_with_legacy_metrics(&pool, WriterOperation::Readiness) .await .expect_err("closed pool acquire errors"); assert!(matches!(closed, sqlx::Error::PoolClosed)); let mut outcomes = BTreeMap::<(String, String), Vec>::new(); for (key, _, _, value) in snapshotter.snapshot().into_vec() { - if key.key().name() != "buzz_db_pool_acquire_wait_seconds" { - continue; - } - let DebugValue::Histogram(samples) = value else { - panic!("pool wait must be a histogram"); - }; let labels = key.key().labels().collect::>(); let label = |name: &str| { labels @@ -472,15 +1246,27 @@ mod tests { .map(|label| label.value().to_owned()) .unwrap_or_default() }; - outcomes.insert( - (label("pool_role"), label("outcome")), - samples - .into_iter() - .map(|sample| sample.into_inner()) - .collect(), - ); + if key.key().name() == "buzz_db_pool_acquire_wait_seconds" { + let DebugValue::Histogram(samples) = value else { + panic!("pool wait must be a histogram"); + }; + if samples.is_empty() { + continue; + } + outcomes.insert( + (label("pool_role"), label("outcome")), + samples + .into_iter() + .map(|sample| sample.into_inner()) + .collect(), + ); + } } - assert!(outcomes.contains_key(&("writer".to_owned(), "success".to_owned()))); + assert!(writer_success); + assert!( + !outcomes.contains_key(&("writer".to_owned(), "cancelled".to_owned())), + "legacy compatibility families must not add a cancellation population" + ); assert!(outcomes.contains_key(&("writer".to_owned(), "error".to_owned()))); let timeout_samples = outcomes .get(&("reader".to_owned(), "timeout".to_owned())) @@ -491,9 +1277,395 @@ mod tests { ); } + async fn deletion_catalog_readiness_records_timeout_and_recovers() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.lock().await; + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(5)) + .connect(&crate::test_support::database_url()) + .await + .expect("connect size-one deletion readiness pool"); + let db = crate::Db::from_pool(pool.clone()); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + let held = pool.acquire().await.expect("hold the only pool connection"); + let timeout = db + .validate_deletion_serving_catalog_for_readiness( + tokio::time::Instant::now() + Duration::from_millis(40), + ) + .await + .expect_err("saturated deletion catalog checkout must time out"); + assert!(matches!( + timeout, + crate::DbError::Sqlx(sqlx::Error::PoolTimedOut) + )); + drop(held); + db.validate_deletion_serving_catalog_for_readiness( + tokio::time::Instant::now() + Duration::from_secs(2), + ) + .await + .expect("deletion catalog readiness must recover after pool release"); + + let snapshot = snapshotter.snapshot().into_vec(); + assert_eq!( + waiter_value(&snapshot, "writer", "readiness"), + Some(0.0), + "deadline terminal must directly balance the readiness waiter" + ); + for outcome in ["timeout", "success"] { + assert!( + snapshot.iter().any(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_acquire_attempts_total" { + return false; + } + let labels = key.key().labels().collect::>(); + let has = |name: &str, expected: &str| { + labels + .iter() + .any(|label| label.key() == name && label.value() == expected) + }; + has("pool_role", "writer") + && has("operation", "readiness") + && has("outcome", outcome) + && matches!(value, DebugValue::Counter(1)) + }), + "missing writer/readiness/{outcome} acquisition terminal" + ); + } + } + + async fn production_db_methods_emit_exact_pool_operation_labels() { + use buzz_core::CommunityId; + use chrono::Utc; + use uuid::Uuid; + + let database_url = crate::test_support::database_url(); + let writer_pool = crate::Db::connect_writer_pool(&crate::DbConfig { + database_url: database_url.clone(), + max_connections: 4, + min_connections: 0, + acquire_timeout_secs: 5, + ..crate::DbConfig::default() + }) + .await + .expect("connect production-method writer pool"); + let reader_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(4) + .acquire_timeout(Duration::from_secs(5)) + .connect(&database_url) + .await + .expect("connect production-method reader pool"); + let writer_db = crate::Db::from_pool(writer_pool.clone()); + let mut routed_db = crate::Db::from_pools(writer_pool.clone(), reader_pool); + routed_db.set_replica_read_max_age_for_tests(Some(Duration::from_secs(5))); + + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let test_scope = CommunityId::from_uuid(Uuid::new_v4()); + let query = crate::EventQuery::for_community(test_scope); + + routed_db.read_pool_boot_ping_for_tests().await; + routed_db + .verify_replica_fence_at_boot() + .await + .expect("real startup fence verification succeeds"); + let _ = crate::replica_fence::probe_once(&writer_pool, routed_db.fence()).await; + routed_db.fence().force_open_for_tests(Utc::now()); + assert_eq!( + writer_db + .readiness_check(tokio::time::Instant::now() + Duration::from_secs(1)) + .await, + crate::DbReadinessOutcome::Success + ); + let _ = writer_db + .lookup_community_by_host("pool-operation-matrix.invalid") + .await; + let _ = writer_db + .lookup_community_by_host_for_management("pool-operation-matrix.invalid") + .await; + let _ = writer_db.list_communities_owned_by(&"a".repeat(64)).await; + let _ = writer_db.lookup_community_host(test_scope).await; + let _ = writer_db + .set_community_icon(test_scope, Some("pool-operation-matrix")) + .await; + let _ = writer_db + .create_community_with_owner( + &format!("pool-operation-matrix-{}.invalid", Uuid::new_v4().simple()), + &"b".repeat(64), + ) + .await; + let _ = writer_db + .archive_community_owned_by( + "pool-operation-matrix.invalid", + &"c".repeat(64), + "protected.invalid", + ) + .await; + let _ = writer_db + .unarchive_community_owned_by("pool-operation-matrix.invalid", &"c".repeat(64)) + .await; + let _ = writer_db.community_of_channel(Uuid::new_v4()).await; + let _ = writer_db.communities_of_channels(&[Uuid::new_v4()]).await; + let _ = writer_db + .ensure_user_for_authorization(test_scope, &[17; 32]) + .await; + let _ = writer_db + .set_agent_owner_for_authorization(test_scope, &[18; 32], &[19; 32]) + .await; + let _ = writer_db.is_pubkey_allowed(test_scope, &[7; 32]).await; + let _ = writer_db + .is_agent_owner(test_scope, &[8; 32], &[9; 32]) + .await; + let _ = writer_db + .moderation_restriction_state(test_scope, &[14; 32]) + .await; + let _ = writer_db + .get_agent_channel_policy(test_scope, &[15; 32]) + .await; + let _ = writer_db + .get_thread_metadata_by_event(test_scope, &[10; 32]) + .await; + let _ = writer_db.get_thread_summary(test_scope, &[16; 32]).await; + let _ = writer_db + .get_channel_for_event_write(test_scope, Uuid::new_v4()) + .await; + let _ = writer_db + .get_members_for_event_write(test_scope, Uuid::new_v4()) + .await; + let _ = writer_db + .get_users_bulk_for_event_write(test_scope, &[vec![11; 32]]) + .await; + let _ = writer_db + .huddle_started_link_exists_for_event_write( + test_scope, + Uuid::new_v4(), + Uuid::new_v4(), + &[12; 32], + ) + .await; + let _ = writer_db + .huddle_started_link_exists(test_scope, Uuid::new_v4(), Uuid::new_v4(), &[13; 32]) + .await; + let _ = writer_db.list_archived(test_scope).await; + let _ = writer_db + .query_events_routed("pool_operation_matrix_writer", &query) + .await; + let write_tx = writer_db + .begin_event_write_transaction() + .await + .expect("event-write semantic entry point begins a real transaction"); + write_tx + .rollback() + .await + .expect("rollback operation-label fixture"); + let _ = writer_db + .is_community_active_for_maintenance(test_scope) + .await; + let _ = writer_db.usage_community_count().await; + let _ = writer_db.reap_expired_ephemeral_channels().await; + let deletion_store = writer_db.deletion_store(); + let _ = deletion_store.reap_expired_serving_write_leases(1).await; + let _ = deletion_store.serving_lease_stats().await; + let _ = routed_db.is_relay_member(test_scope, &"a".repeat(64)).await; + let _ = routed_db + .query_events_routed("pool_operation_matrix_reader", &query) + .await; + routed_db.refresh_pool_waiter_metrics(); + + let snapshot = snapshotter.snapshot().into_vec(); + let attempt_labels = snapshot + .iter() + .filter_map(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_acquire_attempts_total" { + return None; + } + let DebugValue::Counter(value) = value else { + panic!("acquisition attempts must be counters"); + }; + if *value == 0 { + return None; + } + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + assert_eq!(labels.get("outcome").map(String::as_str), Some("success")); + Some((labels["pool_role"].clone(), labels["operation"].clone())) + }) + .collect::>(); + let expected = super::POOL_ACQUIRE_VALID_PAIRS + .into_iter() + .map(|(pool_role, operation)| (pool_role.to_owned(), operation.to_owned())) + .collect::>(); + assert_eq!( + attempt_labels, expected, + "real production Db/store methods must emit every exact valid operation pair" + ); + + let duration_labels = snapshot + .iter() + .filter_map(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_acquire_duration_seconds" { + return None; + } + let DebugValue::Histogram(samples) = value else { + panic!("acquisition duration must be a histogram"); + }; + assert!(!samples.is_empty()); + let labels = key + .key() + .labels() + .map(|label| (label.key(), label.value())) + .collect::>(); + assert!(!labels.contains_key("outcome")); + Some(( + labels["pool_role"].to_owned(), + labels["operation"].to_owned(), + )) + }) + .collect::>(); + assert_eq!(duration_labels, expected); + + let waiter_labels = snapshot + .iter() + .filter_map(|(key, _, _, value)| { + if key.key().name() != "buzz_db_pool_waiters" { + return None; + } + let DebugValue::Gauge(value) = value else { + panic!("pool waiters must be gauges"); + }; + assert_eq!(value.into_inner(), 0.0); + let labels = key + .key() + .labels() + .map(|label| (label.key(), label.value())) + .collect::>(); + Some(( + labels["pool_role"].to_owned(), + labels["operation"].to_owned(), + )) + }) + .collect::>(); + assert_eq!(waiter_labels, expected); + } + + async fn serving_write_gate_records_cancel_timeout_success_and_recovery() { + let _test_guard = super::POOL_METRICS_TEST_LOCK.lock().await; + let database_url = crate::test_support::database_url(); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + // The same budget also covers the pool's initial physical + // connection. Keep enough headroom for a cold CI database; the + // held size-one connection below still deterministically drives + // the checkout timeout terminal. + .acquire_timeout(Duration::from_secs(1)) + .connect(&database_url) + .await + .expect("connect size-one serving-write test pool"); + let db = crate::Db::from_pool(pool.clone()); + if std::env::var("BUZZ_TEST_SCHEMA_MODE").as_deref() != Ok("desired") { + db.migrate().await.expect("migrate serving-write test DB"); + } + let test_scope = db + .ensure_configured_community(&format!( + "pool-observability-{}.example", + uuid::Uuid::new_v4().simple() + )) + .await + .expect("create serving-write test community") + .id; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + let held = pool.acquire().await.expect("hold sole writer connection"); + + let store = db.deletion_store(); + let mut cancelled = Box::pin(store.is_serving_active(test_scope)); + tokio::select! { + result = &mut cancelled => panic!("blocked serving-write gate unexpectedly completed: {result:?}"), + () = tokio::time::sleep(Duration::from_millis(25)) => {} + } + assert_eq!( + waiter_value(&snapshotter.snapshot().into_vec(), "writer", "event_write"), + Some(1.0) + ); + drop(cancelled); + + let timeout = store + .is_serving_active(test_scope) + .await + .expect_err("saturated serving-write gate times out"); + assert!(matches!( + timeout, + crate::DbError::Sqlx(sqlx::Error::PoolTimedOut) + )); + drop(held); + + assert!(store + .is_serving_active(test_scope) + .await + .expect("serving-write gate recovers after release")); + let lease = store + .acquire_serving_write_lease( + test_scope, + "pool_observability", + "pool-observability-test", + Duration::from_secs(5), + ) + .await + .expect("serving-write lease acquires through event-write seam"); + assert!(store + .release_serving_write_lease(&lease) + .await + .expect("serving-write lease release")); + refresh_pool_waiters(false); + + let snapshot = snapshotter.snapshot().into_vec(); + assert_eq!(waiter_value(&snapshot, "writer", "event_write"), Some(0.0)); + assert_eq!(attempt_count(&snapshot, "event_write", "cancelled"), 1); + assert_eq!(attempt_count(&snapshot, "event_write", "timeout"), 1); + assert!( + attempt_count(&snapshot, "event_write", "success") >= 3, + "gate recovery plus lease acquire/release must emit successes" + ); + } + + fn attempt_count( + snapshot: &[( + metrics_util::CompositeKey, + Option, + Option, + DebugValue, + )], + operation: &str, + outcome: &str, + ) -> u64 { + snapshot + .iter() + .find_map(|(key, _, _, value)| { + let labels = key.key().labels().collect::>(); + (key.key().name() == "buzz_db_pool_acquire_attempts_total" + && labels + .iter() + .any(|label| label.key() == "operation" && label.value() == operation) + && labels + .iter() + .any(|label| label.key() == "outcome" && label.value() == outcome)) + .then(|| match value { + DebugValue::Counter(value) => *value, + _ => panic!("pool attempts must be a counter"), + }) + }) + .unwrap_or(0) + } + async fn advisory_lock_records_success_contention_timeout_and_error() { - let database_url = std::env::var("TEST_DATABASE_URL") - .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_owned()); // sadscan:disable np.postgres.1 -- local test-only credentials + let database_url = crate::test_support::database_url(); let pool = sqlx::postgres::PgPoolOptions::new() .max_connections(4) .connect(&database_url) @@ -640,6 +1812,24 @@ mod tests { super::pool_acquire_records_success_timeout_and_error_with_wait_time().await; } + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn production_db_methods_emit_exact_pool_operation_labels() { + super::production_db_methods_emit_exact_pool_operation_labels().await; + } + + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn deletion_catalog_readiness_records_timeout_and_recovers() { + super::deletion_catalog_readiness_records_timeout_and_recovers().await; + } + + #[tokio::test(flavor = "current_thread")] + #[ignore = "requires Postgres"] + async fn serving_write_gate_records_cancel_timeout_success_and_recovery() { + super::serving_write_gate_records_cancel_timeout_success_and_recovery().await; + } + #[tokio::test(flavor = "current_thread")] #[ignore = "requires Postgres"] async fn advisory_lock_records_success_contention_timeout_and_error() { diff --git a/crates/buzz-db/src/runtime/replica_fence.rs b/crates/buzz-db/src/runtime/replica_fence.rs index 2194c8bb30a..044dc3a58c6 100644 --- a/crates/buzz-db/src/runtime/replica_fence.rs +++ b/crates/buzz-db/src/runtime/replica_fence.rs @@ -395,7 +395,12 @@ pub async fn verify_floor_guard_behavior(pool: &PgPool) -> crate::Result<()> { } }; - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // 1. Pool arming (Perci: assert the effective value, not the intent). let armed: String = sqlx::query_scalar("SHOW buzz.created_at_floor") @@ -552,7 +557,11 @@ pub enum ProbeError { /// a single SELECT would not guarantee evaluation order across the /// subexpressions, reopening the race this ordering exists to close. async fn sample_writer(writer: &PgPool) -> Result { - let mut conn = writer.acquire().await?; + let mut conn = crate::observability::acquire_writer( + writer, + crate::observability::WriterOperation::Maintenance, + ) + .await?; // 1. S first. let sampled_at: DateTime = sqlx::query_scalar("SELECT clock_timestamp()") @@ -671,11 +680,16 @@ pub async fn probe_once(writer: &PgPool, fence: &ReplicaFence) -> Result Db { Db::from_pool(pool) } +#[tokio::test] +async fn begin_transaction_compatibility_alias_is_preserved() { + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy(&crate::test_support::database_url()) + .expect("construct lazy compatibility pool"); + pool.close().await; + let db = Db::from_pool(pool); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + #[allow(deprecated)] + let result = db.begin_transaction().await; + assert!(matches!( + result, + Err(DbError::Sqlx(sqlx::Error::PoolClosed)) + )); + + let counters = snapshotter + .snapshot() + .into_vec() + .into_iter() + .filter_map(|(key, _, _, value)| { + let name = key.key().name(); + if ![ + "buzz_db_pool_acquire_attempts_total", + "buzz_db_pool_acquisitions_total", + ] + .contains(&name) + { + return None; + } + let DebugValue::Counter(value) = value else { + panic!("pool acquisition terminals must be counters"); + }; + let labels = key + .key() + .labels() + .map(|label| (label.key().to_owned(), label.value().to_owned())) + .collect::>(); + Some(((name.to_owned(), labels), value)) + }) + .collect::>(); + let expected = [ + ( + ( + "buzz_db_pool_acquire_attempts_total".to_owned(), + [ + ("operation".to_owned(), "event_write".to_owned()), + ("outcome".to_owned(), "error".to_owned()), + ("pool_role".to_owned(), "writer".to_owned()), + ] + .into_iter() + .collect(), + ), + 1, + ), + ( + ( + "buzz_db_pool_acquisitions_total".to_owned(), + [ + ("outcome".to_owned(), "error".to_owned()), + ("pool_role".to_owned(), "writer".to_owned()), + ] + .into_iter() + .collect(), + ), + 1, + ), + ] + .into_iter() + .collect::>(); + assert_eq!(counters, expected); +} + +#[test] +fn nip43_reconciliation_compatibility_alias_is_preserved() { + #[allow(deprecated)] + async fn call( + db: &Db, + community_id: CommunityId, + relay_pubkey: &nostr::PublicKey, + ) -> crate::Result { + db.nip43_membership_snapshot_needs_reconciliation(community_id, relay_pubkey) + .await + } + + let _ = call; +} + #[tokio::test] #[ignore = "requires Postgres"] async fn readiness_check_distinguishes_pool_exhaustion_from_success() { diff --git a/crates/buzz-db/src/store/allowlist.rs b/crates/buzz-db/src/store/allowlist.rs index d2780a498c0..2b72780a621 100644 --- a/crates/buzz-db/src/store/allowlist.rs +++ b/crates/buzz-db/src/store/allowlist.rs @@ -28,12 +28,17 @@ impl Db { /// Check if a pubkey is in the allowlist for `community`. #[datastore_span(name = "is_pubkey_allowed", system = "postgresql")] pub async fn is_pubkey_allowed(&self, community: CommunityId, pubkey: &[u8]) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authentication, + ) + .await?; let row = sqlx::query( "SELECT COUNT(*) as cnt FROM pubkey_allowlist WHERE community_id = $1 AND pubkey = $2", ) .bind(community.as_uuid()) .bind(pubkey) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; let cnt: i64 = row.try_get("cnt")?; Ok(cnt > 0) @@ -42,10 +47,15 @@ impl Db { /// Check if the community allowlist has any entries (i.e. is enforcement active). #[datastore_span(name = "has_allowlist_entries", system = "postgresql")] pub async fn has_allowlist_entries(&self, community: CommunityId) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authentication, + ) + .await?; let row = sqlx::query("SELECT COUNT(*) as cnt FROM pubkey_allowlist WHERE community_id = $1") .bind(community.as_uuid()) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; let cnt: i64 = row.try_get("cnt")?; Ok(cnt > 0) @@ -60,6 +70,11 @@ impl Db { added_by: &[u8], note: Option<&str>, ) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let result = sqlx::query( "INSERT INTO pubkey_allowlist (community_id, pubkey, added_by, note) VALUES ($1, $2, $3, $4) \ ON CONFLICT DO NOTHING", @@ -68,7 +83,7 @@ impl Db { .bind(pubkey) .bind(added_by) .bind(note) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) } @@ -80,11 +95,16 @@ impl Db { community: CommunityId, pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let result = sqlx::query("DELETE FROM pubkey_allowlist WHERE community_id = $1 AND pubkey = $2") .bind(community.as_uuid()) .bind(pubkey) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) } @@ -92,11 +112,16 @@ impl Db { /// List all pubkeys in the community allowlist. #[datastore_span(name = "list_allowlist", system = "postgresql")] pub async fn list_allowlist(&self, community: CommunityId) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( "SELECT pubkey, added_by, added_at, note FROM pubkey_allowlist WHERE community_id = $1 ORDER BY added_at DESC", ) .bind(community.as_uuid()) - .fetch_all(&self.pool) + .fetch_all(&mut *connection) .await?; let mut out = Vec::with_capacity(rows.len()); diff --git a/crates/buzz-db/src/store/archived_identities.rs b/crates/buzz-db/src/store/archived_identities.rs index 102ca44f96f..b1636de31a3 100644 --- a/crates/buzz-db/src/store/archived_identities.rs +++ b/crates/buzz-db/src/store/archived_identities.rs @@ -34,11 +34,16 @@ pub struct ArchivedIdentity { /// Returns `true` if `pubkey` (64-char hex) is archived in `community_id`. pub async fn is_archived(pool: &PgPool, community_id: CommunityId, pubkey: &str) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let row = sqlx::query("SELECT 1 FROM archived_identities WHERE community_id = $1 AND pubkey = $2") .bind(community_id.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.is_some()) } @@ -59,6 +64,11 @@ pub async fn archive( replaced_by: Option<&str>, request_event_id: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( "INSERT INTO archived_identities \ (community_id, pubkey, consent_path, actor, reason, replaced_by, request_event_id) \ @@ -72,7 +82,7 @@ pub async fn archive( .bind(reason) .bind(replaced_by) .bind(request_event_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -83,11 +93,16 @@ pub async fn archive( /// Returns `true` if a row was deleted, `false` if the identity was not archived /// in that community. pub async fn unarchive(pool: &PgPool, community_id: CommunityId, pubkey: &str) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query("DELETE FROM archived_identities WHERE community_id = $1 AND pubkey = $2") .bind(community_id.as_uuid()) .bind(pubkey) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -98,12 +113,17 @@ pub async fn list_archived( pool: &PgPool, community_id: CommunityId, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let rows = sqlx::query( "SELECT pubkey, consent_path, actor, reason, replaced_by, request_event_id, archived_at \ FROM archived_identities WHERE community_id = $1 ORDER BY archived_at ASC", ) .bind(community_id.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() diff --git a/crates/buzz-db/src/store/channel.rs b/crates/buzz-db/src/store/channel.rs index c581f529f72..5e89b1101b6 100644 --- a/crates/buzz-db/src/store/channel.rs +++ b/crates/buzz-db/src/store/channel.rs @@ -29,6 +29,27 @@ pub use crate::channel_members::{ LargeChannelRoster, LockedMemberSnapshot, MemberRecord, UserRecord, }; +async fn begin_event_write_transaction( + pool: &PgPool, +) -> Result> { + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + Ok(sqlx::Transaction::begin(connection, None).await?) +} + +async fn acquire_event_write_connection( + pool: &PgPool, +) -> Result> { + Ok(crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?) +} + /// A channel row as returned from the database. #[derive(Debug, Clone)] pub struct ChannelRecord { @@ -104,7 +125,7 @@ pub async fn create_channel( let id = Uuid::new_v4(); - let mut tx = pool.begin().await?; + let mut tx = begin_event_write_transaction(pool).await?; sqlx::query( r#" @@ -197,7 +218,7 @@ pub async fn create_channel_with_id( return Err(DbError::InvalidData("channel name is required".into())); } - let mut tx = pool.begin().await?; + let mut tx = begin_event_write_transaction(pool).await?; let rows_affected = sqlx::query( r#" @@ -269,6 +290,22 @@ pub async fn get_channel( community_id: CommunityId, channel_id: Uuid, ) -> Result { + get_channel_with_operation( + pool, + community_id, + channel_id, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +async fn get_channel_with_operation( + pool: &PgPool, + community_id: CommunityId, + channel_id: Uuid, + operation: crate::observability::WriterOperation, +) -> Result { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let row = sqlx::query( r#" SELECT id, name, channel_type::text AS channel_type, visibility::text AS visibility, @@ -283,7 +320,7 @@ pub async fn get_channel( ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await? .ok_or(DbError::ChannelNotFound(channel_id))?; @@ -334,6 +371,22 @@ pub async fn list_channels( community_id: CommunityId, visibility: Option<&str>, ) -> Result> { + list_channels_with_operation( + pool, + community_id, + visibility, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +async fn list_channels_with_operation( + pool: &PgPool, + community_id: CommunityId, + visibility: Option<&str>, + operation: crate::observability::WriterOperation, +) -> Result> { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let rows = if let Some(vis) = visibility { sqlx::query( r#" @@ -352,7 +405,7 @@ pub async fn list_channels( ) .bind(community_id.as_uuid()) .bind(vis) - .fetch_all(pool) + .fetch_all(&mut *connection) .await? } else { sqlx::query( @@ -371,7 +424,7 @@ pub async fn list_channels( "#, ) .bind(community_id.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await? }; @@ -530,7 +583,7 @@ pub async fn update_channel( // this transition — whose own deadline reset is then the latest word. // Non-TTL updates don't touch the fast path and skip the lock. if updates.ttl_seconds.is_some() { - let mut tx = pool.begin().await?; + let mut tx = begin_event_write_transaction(pool).await?; sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") .bind(format!( "buzz_channel_ttl:{}:{}", @@ -545,13 +598,20 @@ pub async fn update_channel( } tx.commit().await?; } else { - let result = q.execute(pool).await?; + let mut connection = acquire_event_write_connection(pool).await?; + let result = q.execute(&mut *connection).await?; if result.rows_affected() == 0 { return Err(DbError::ChannelNotFound(channel_id)); } } - get_channel(pool, community_id, channel_id).await + get_channel_with_operation( + pool, + community_id, + channel_id, + crate::observability::WriterOperation::EventWrite, + ) + .await } /// Sets the topic for a channel, recording who set it and when. @@ -562,6 +622,7 @@ pub async fn set_topic( topic: &str, set_by: &[u8], ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; let result = sqlx::query( "UPDATE channels SET topic = $1, topic_set_by = $2, topic_set_at = NOW() \ WHERE community_id = $3 AND id = $4 AND deleted_at IS NULL", @@ -570,7 +631,7 @@ pub async fn set_topic( .bind(set_by) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() == 0 { return Err(DbError::ChannelNotFound(channel_id)); @@ -586,6 +647,7 @@ pub async fn set_purpose( purpose: &str, set_by: &[u8], ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; let result = sqlx::query( "UPDATE channels SET purpose = $1, purpose_set_by = $2, purpose_set_at = NOW() \ WHERE community_id = $3 AND id = $4 AND deleted_at IS NULL", @@ -594,7 +656,7 @@ pub async fn set_purpose( .bind(set_by) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() == 0 { return Err(DbError::ChannelNotFound(channel_id)); @@ -611,13 +673,14 @@ pub async fn archive_channel( community_id: CommunityId, channel_id: Uuid, ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; // First check: does the channel exist and what is its state? let row = sqlx::query( "SELECT archived_at FROM channels WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -638,7 +701,7 @@ pub async fn archive_channel( ) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(()) @@ -653,13 +716,14 @@ pub async fn unarchive_channel( community_id: CommunityId, channel_id: Uuid, ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; // First check: does the channel exist and what is its state? let row = sqlx::query( "SELECT archived_at FROM channels WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -682,7 +746,7 @@ pub async fn unarchive_channel( ) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(()) @@ -697,12 +761,13 @@ pub async fn soft_delete_channel( community_id: CommunityId, channel_id: Uuid, ) -> Result { + let mut connection = acquire_event_write_connection(pool).await?; let result = sqlx::query( "UPDATE channels SET deleted_at = NOW() WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", ) .bind(community_id.as_uuid()) .bind(channel_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -714,6 +779,11 @@ pub async fn soft_delete_channel( /// `archived_at IS NULL` guard prevents double-archiving even if called /// concurrently from multiple relay pods. pub async fn reap_expired_ephemeral_channels(pool: &PgPool) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let rows = sqlx::query( "UPDATE channels AS ch SET archived_at = NOW() \ FROM communities AS c \ @@ -726,7 +796,7 @@ pub async fn reap_expired_ephemeral_channels(pool: &PgPool) -> Result Result { + get_channel_with_operation( + &self.pool, + community_id, + channel_id, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Returns the canvas content for a channel, if any. #[datastore_span(name = "get_canvas", system = "postgresql")] pub async fn get_canvas( @@ -841,6 +928,22 @@ impl Db { list_channels(&self.pool, community_id, visibility).await } + /// Lists channels during startup reconciliation. + #[datastore_span(name = "list_channels_for_bootstrap", system = "postgresql")] + pub async fn list_channels_for_bootstrap( + &self, + community_id: CommunityId, + visibility: Option<&str>, + ) -> Result> { + list_channels_with_operation( + &self.pool, + community_id, + visibility, + crate::observability::WriterOperation::Bootstrap, + ) + .await + } + /// Updates a channel's name and/or description. #[datastore_span(name = "update_channel", system = "postgresql")] pub async fn update_channel( diff --git a/crates/buzz-db/src/store/channel_members.rs b/crates/buzz-db/src/store/channel_members.rs index dd526a0a46c..8280ca01f82 100644 --- a/crates/buzz-db/src/store/channel_members.rs +++ b/crates/buzz-db/src/store/channel_members.rs @@ -100,7 +100,12 @@ pub async fn verify_channel_roster_fence_catalog<'e>( /// function. This rolled-back probe verifies that a canonical empty roster is /// accepted while a stale roster member is rejected with `check_violation`. pub async fn verify_channel_roster_fence_behavior(pool: &sqlx::PgPool) -> Result<()> { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let community_id = Uuid::new_v4(); let channel_id = Uuid::new_v4(); sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") @@ -323,7 +328,12 @@ pub async fn lock_member_snapshot( channel_id: Uuid, relay_pubkey: &[u8], ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // Match the canonical replacement writer's lock order. Old binaries take // this key before INSERT; migration 0032 then takes the membership key in // the INSERT trigger. Taking both in that order avoids mixed-version @@ -396,7 +406,12 @@ pub async fn add_member( ))); } - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // First statement: serialize the whole role-check / owner-count / upsert // sequence against concurrent membership writes on this channel. @@ -577,7 +592,12 @@ pub async fn remove_member( crate::user::is_agent_owner(pool, community_id, pubkey, actor_pubkey).await? }; - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // First statement: serialize the actor-role check, the last-owner count and // the UPDATE against concurrent membership writes on this channel (same key @@ -648,6 +668,11 @@ pub async fn is_member( channel_id: Uuid, pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT COUNT(*) as cnt FROM channel_members cm \ JOIN channels c ON cm.community_id = c.community_id AND cm.channel_id = c.id AND c.deleted_at IS NULL \ @@ -656,7 +681,7 @@ pub async fn is_member( .bind(community_id.as_uuid()) .bind(channel_id) .bind(pubkey) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; let cnt: i64 = row.try_get("cnt")?; Ok(cnt > 0) @@ -674,6 +699,11 @@ pub async fn membership_pairs( if channel_ids.is_empty() || pubkeys.is_empty() { return Ok(Vec::new()); } + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( "SELECT cm.channel_id, cm.pubkey FROM channel_members cm \ JOIN channels c ON cm.community_id = c.community_id AND cm.channel_id = c.id AND c.deleted_at IS NULL \ @@ -682,7 +712,7 @@ pub async fn membership_pairs( .bind(community_id.as_uuid()) .bind(channel_ids) .bind(pubkeys) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() .map(|row| Ok((row.try_get("channel_id")?, row.try_get("pubkey")?))) @@ -702,6 +732,22 @@ pub async fn get_members( community_id: CommunityId, channel_id: Uuid, ) -> Result> { + get_members_with_operation( + pool, + community_id, + channel_id, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +async fn get_members_with_operation( + pool: &PgPool, + community_id: CommunityId, + channel_id: Uuid, + operation: crate::observability::WriterOperation, +) -> Result> { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let rows = sqlx::query( r#" SELECT cm.channel_id, cm.pubkey, cm.role::text AS role, cm.joined_at, cm.invited_by, cm.removed_at @@ -713,7 +759,7 @@ pub async fn get_members( ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter().map(row_to_member_record).collect() } @@ -733,6 +779,11 @@ pub async fn get_members_bulk( if channel_ids.is_empty() { return Ok(Vec::new()); } + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( r#" SELECT cm.channel_id, cm.pubkey, cm.role::text AS role, cm.joined_at, cm.invited_by, cm.removed_at @@ -744,7 +795,7 @@ pub async fn get_members_bulk( ) .bind(community_id.as_uuid()) .bind(channel_ids) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter().map(row_to_member_record).collect() } @@ -758,6 +809,11 @@ pub async fn get_accessible_channel_ids( community_id: CommunityId, pubkey: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( r#" SELECT cm.channel_id @@ -772,7 +828,7 @@ pub async fn get_accessible_channel_ids( ) .bind(community_id.as_uuid()) .bind(pubkey) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() @@ -806,6 +862,11 @@ pub async fn list_large_channel_rosters_needing_reconciliation( minimum_members: i64, relay_pubkey: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let rows = sqlx::query( r#" WITH large_rosters AS ( @@ -843,7 +904,7 @@ pub async fn list_large_channel_rosters_needing_reconciliation( ) .bind(minimum_members) .bind(relay_pubkey) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() @@ -964,6 +1025,11 @@ pub async fn get_accessible_channels( visibility_filter: Option<&str>, member_only: Option, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; // When `member_only` is `Some(true)`, restrict to channels where the user // has an active membership (cm.channel_id IS NOT NULL). This is a strict // subset of the default result set and is pushed into SQL so the LIMIT 1000 @@ -1008,7 +1074,7 @@ pub async fn get_accessible_channels( query }; - let rows = query.fetch_all(pool).await?; + let rows = query.fetch_all(&mut *connection).await?; rows.into_iter() .map(|row| { let is_member: bool = row.try_get("is_member").unwrap_or(false); @@ -1027,6 +1093,11 @@ pub async fn get_bot_members( pool: &PgPool, community_id: CommunityId, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( r#" SELECT cm.pubkey, u.display_name, u.agent_type, u.capabilities, @@ -1040,7 +1111,7 @@ pub async fn get_bot_members( "#, ) .bind(community_id.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; let mut out = Vec::with_capacity(rows.len()); @@ -1071,10 +1142,26 @@ pub async fn get_users_bulk( pool: &PgPool, community_id: CommunityId, pubkeys: &[Vec], +) -> Result> { + get_users_bulk_with_operation( + pool, + community_id, + pubkeys, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await +} + +async fn get_users_bulk_with_operation( + pool: &PgPool, + community_id: CommunityId, + pubkeys: &[Vec], + operation: crate::observability::WriterOperation, ) -> Result> { if pubkeys.is_empty() { return Ok(Vec::new()); } + let mut connection = crate::observability::acquire_writer(pool, operation).await?; // Build a parameterised IN clause: ($2, $3, ...); $1 is community_id. let placeholders = (2..(pubkeys.len() + 2)) @@ -1091,7 +1178,7 @@ pub async fn get_users_bulk( q = q.bind(pk); } - let rows = q.fetch_all(pool).await?; + let rows = q.fetch_all(&mut *connection).await?; let mut out = Vec::with_capacity(rows.len()); for row in rows { @@ -1124,12 +1211,17 @@ pub async fn get_member_count( community_id: CommunityId, channel_id: Uuid, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT COUNT(*) as cnt FROM channel_members WHERE community_id = $1 AND channel_id = $2 AND removed_at IS NULL", ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; Ok(row.try_get("cnt")?) } @@ -1146,6 +1238,11 @@ pub async fn get_member_counts_bulk( if channel_ids.is_empty() { return Ok(std::collections::HashMap::new()); } + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( "SELECT channel_id, COUNT(*) as cnt FROM channel_members \ @@ -1159,7 +1256,7 @@ pub async fn get_member_counts_bulk( } qb.push(") GROUP BY channel_id"); - let rows = qb.build().fetch_all(pool).await?; + let rows = qb.build().fetch_all(&mut *connection).await?; let mut map = std::collections::HashMap::with_capacity(rows.len()); for row in rows { @@ -1179,6 +1276,11 @@ pub async fn get_member_role( channel_id: Uuid, pubkey: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT cm.role::text AS role FROM channel_members cm \ JOIN channels c ON cm.community_id = c.community_id AND cm.channel_id = c.id AND c.deleted_at IS NULL \ @@ -1187,7 +1289,7 @@ pub async fn get_member_role( .bind(community_id.as_uuid()) .bind(channel_id) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.map(|r| r.try_get("role")).transpose()?) } @@ -1196,7 +1298,14 @@ impl Db { /// Verify the mixed-version channel-roster database fence end to end. #[datastore_span(name = "verify_channel_roster_fence", system = "postgresql")] pub async fn verify_channel_roster_fence(&self) -> Result<()> { - verify_channel_roster_fence_catalog(&self.pool).await?; + { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + verify_channel_roster_fence_catalog(&mut *connection).await?; + } verify_channel_roster_fence_behavior(&self.pool).await } @@ -1277,6 +1386,22 @@ impl Db { get_members(&self.pool, community_id, channel_id).await } + /// Return a channel roster used to build or validate an event mutation. + #[datastore_span(name = "get_members_for_event_write", system = "postgresql")] + pub async fn get_members_for_event_write( + &self, + community_id: CommunityId, + channel_id: Uuid, + ) -> Result> { + get_members_with_operation( + &self.pool, + community_id, + channel_id, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Returns active members for multiple channels in a single query. #[datastore_span(name = "get_members_bulk", system = "postgresql")] pub async fn get_members_bulk( @@ -1346,6 +1471,22 @@ impl Db { get_users_bulk(&self.pool, community_id, pubkeys).await } + /// Bulk-fetch user names while constructing an event and its mention tags. + #[datastore_span(name = "get_users_bulk_for_event_write", system = "postgresql")] + pub async fn get_users_bulk_for_event_write( + &self, + community_id: CommunityId, + pubkeys: &[Vec], + ) -> Result> { + get_users_bulk_with_operation( + &self.pool, + community_id, + pubkeys, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Returns the count of active members in a channel. #[datastore_span(name = "get_member_count", system = "postgresql")] pub async fn get_member_count( @@ -2944,6 +3085,37 @@ mod postgres_tests { drop_scratch_db(&admin, pool, &scratch_name).await; } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn channel_roster_fence_verification_supports_size_one_pool() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (seed_pool, scratch_name) = create_scratch_db(&admin, "roster_fence_size_one").await; + seed_pool.close().await; + + let base_url = admin_url().await; + let path = base_url.rfind('/').expect("database URL path"); + let scratch_url = format!("{}/{}", &base_url[..path], scratch_name); + let pool = PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(1)) + .connect(&scratch_url) + .await + .expect("connect size-one writer pool"); + let db = Db::from_pool(pool.clone()); + + tokio::time::timeout( + std::time::Duration::from_secs(2), + db.verify_channel_roster_fence(), + ) + .await + .expect("roster verification must not self-deadlock on its second checkout") + .expect("migrated roster fence verifies on a size-one pool"); + + drop_scratch_db(&admin, pool, &scratch_name).await; + } + #[tokio::test] #[ignore = "requires Postgres"] async fn desired_schema_rejects_stale_legacy_roster_role() { diff --git a/crates/buzz-db/src/store/community.rs b/crates/buzz-db/src/store/community.rs index dd8a2e58bc8..bcd38f4e5ce 100644 --- a/crates/buzz-db/src/store/community.rs +++ b/crates/buzz-db/src/store/community.rs @@ -91,6 +91,11 @@ impl Db { &self, normalized_host: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::TenantResolution, + ) + .await?; let row = sqlx::query( r#" SELECT id, host @@ -102,7 +107,7 @@ impl Db { "#, ) .bind(normalized_host) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { @@ -120,11 +125,37 @@ impl Db { /// Returns whether a community id still exists in the active lifecycle state. #[datastore_span(name = "is_community_active", system = "postgresql")] pub async fn is_community_active(&self, community_id: CommunityId) -> Result { + self.is_community_active_with_operation( + community_id, + crate::observability::WriterOperation::Authorization, + ) + .await + } + + /// Background lifecycle revalidation variant of [`Self::is_community_active`]. + #[datastore_span(name = "is_community_active_for_maintenance", system = "postgresql")] + pub async fn is_community_active_for_maintenance( + &self, + community_id: CommunityId, + ) -> Result { + self.is_community_active_with_operation( + community_id, + crate::observability::WriterOperation::Maintenance, + ) + .await + } + + async fn is_community_active_with_operation( + &self, + community_id: CommunityId, + operation: crate::observability::WriterOperation, + ) -> Result { + let mut connection = crate::observability::acquire_writer(&self.pool, operation).await?; let active = sqlx::query_scalar::<_, bool>( "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", ) .bind(community_id.as_uuid()) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; Ok(active) } @@ -138,9 +169,14 @@ impl Db { &self, normalized_host: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query("SELECT id, host FROM communities WHERE lower(host) = lower($1)") .bind(normalized_host) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { Ok(CommunityRecord { @@ -161,6 +197,11 @@ impl Db { owner_pubkey: &str, ) -> Result> { let owner_pubkey = owner_pubkey.to_ascii_lowercase(); + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( r#" SELECT c.id, c.host, c.created_at, c.archived_at @@ -172,7 +213,7 @@ impl Db { "#, ) .bind(owner_pubkey) - .fetch_all(&self.pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() @@ -203,6 +244,11 @@ impl Db { /// is never used to re-derive the community. #[datastore_span(name = "lookup_community_host", system = "postgresql")] pub async fn lookup_community_host(&self, community_id: CommunityId) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::TenantResolution, + ) + .await?; let row = sqlx::query( r#" SELECT host @@ -214,7 +260,7 @@ impl Db { "#, ) .bind(community_id.as_uuid()) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { @@ -255,6 +301,11 @@ impl Db { community_id: CommunityId, icon: Option<&str>, ) -> Result<()> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; sqlx::query( r#" UPDATE communities @@ -264,7 +315,7 @@ impl Db { ) .bind(community_id.as_uuid()) .bind(icon) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(()) } @@ -279,6 +330,35 @@ impl Db { &self, normalized_host: &str, ) -> Result { + self.ensure_configured_community_with_operation( + normalized_host, + crate::observability::WriterOperation::Authorization, + ) + .await + } + + /// Ensure the deployment-configured community during process bootstrap. + #[datastore_span( + name = "ensure_configured_community_for_bootstrap", + system = "postgresql" + )] + pub async fn ensure_configured_community_for_bootstrap( + &self, + normalized_host: &str, + ) -> Result { + self.ensure_configured_community_with_operation( + normalized_host, + crate::observability::WriterOperation::Bootstrap, + ) + .await + } + + async fn ensure_configured_community_with_operation( + &self, + normalized_host: &str, + operation: crate::observability::WriterOperation, + ) -> Result { + let mut connection = crate::observability::acquire_writer(&self.pool, operation).await?; let row = sqlx::query( r#" INSERT INTO communities (host) @@ -290,7 +370,7 @@ impl Db { "#, ) .bind(normalized_host) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await? .ok_or_else(|| { DbError::AccessDenied(format!( @@ -321,7 +401,12 @@ impl Db { owner_pubkey: &str, ) -> Result { let owner_pubkey = owner_pubkey.to_ascii_lowercase(); - let mut tx = self.pool.begin().await?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // Serialize on the owner pubkey so concurrent creates to the same // owner cannot both pass the ownership count check. @@ -412,6 +497,11 @@ impl Db { owner_pubkey: &str, protected_deployment_host: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( r#"UPDATE communities c SET archived_at = COALESCE(c.archived_at, now()) @@ -428,7 +518,7 @@ impl Db { .bind(normalized_host) .bind(owner_pubkey) .bind(protected_deployment_host) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { Ok(ArchivedCommunityRecord { @@ -447,6 +537,11 @@ impl Db { normalized_host: &str, owner_pubkey: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( r#"UPDATE communities c SET archived_at = NULL @@ -461,7 +556,7 @@ impl Db { ) .bind(normalized_host) .bind(owner_pubkey) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { Ok(UnarchivedCommunityRecord { @@ -478,6 +573,11 @@ impl Db { /// they are acting on, rather than falling back to an implicit default. #[datastore_span(name = "community_of_channel", system = "postgresql")] pub async fn community_of_channel(&self, channel_id: Uuid) -> Result> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::TenantResolution, + ) + .await?; let row = sqlx::query( r#" SELECT community_id @@ -487,7 +587,7 @@ impl Db { "#, ) .bind(channel_id) - .fetch_optional(&self.pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| { @@ -523,6 +623,11 @@ impl Db { if channel_ids.is_empty() { return Ok(std::collections::HashMap::new()); } + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; let rows = sqlx::query( r#" SELECT id, community_id @@ -532,7 +637,7 @@ impl Db { "#, ) .bind(channel_ids) - .fetch_all(&self.pool) + .fetch_all(&mut *connection) .await?; let mut out = std::collections::HashMap::with_capacity(rows.len()); diff --git a/crates/buzz-db/src/store/deletion.rs b/crates/buzz-db/src/store/deletion.rs index d34b39f14b1..0e184e00d88 100644 --- a/crates/buzz-db/src/store/deletion.rs +++ b/crates/buzz-db/src/store/deletion.rs @@ -631,7 +631,38 @@ pub struct DeletionStore { impl Db { /// Validate the minimum deletion fence catalog required by serving paths. pub async fn validate_deletion_serving_catalog(&self) -> Result<()> { - self.deletion_store().validate_serving_catalog().await + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + self.deletion_store() + .validate_serving_catalog_on(&mut connection) + .await + } + + /// Validate the serving catalog inside the readiness request's absolute + /// deadline, attributing only the one real writer checkout to readiness. + pub async fn validate_deletion_serving_catalog_for_readiness( + &self, + deadline: tokio::time::Instant, + ) -> Result<()> { + let mut connection = crate::observability::acquire_writer_until( + &self.pool, + crate::observability::WriterOperation::Readiness, + deadline, + ) + .await?; + match tokio::time::timeout_at( + deadline, + self.deletion_store() + .validate_serving_catalog_on(&mut connection), + ) + .await + { + Err(_) => Err(sqlx::Error::PoolTimedOut.into()), + Ok(result) => result, + } } /// Validate the exact live community-deletion tenant catalog for destruction. @@ -784,13 +815,22 @@ impl DeletionStore { /// Validate the deletion catalog contract required by relay serving. pub async fn validate_serving_catalog(&self) -> Result<()> { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; + self.validate_serving_catalog_on(&mut connection).await + } + + async fn validate_serving_catalog_on(&self, conn: &mut PgConnection) -> Result<()> { let runtime_columns = sqlx::query( "SELECT attname, format_type(atttypid, atttypmod) AS type_name, attnotnull \ FROM pg_attribute WHERE attrelid = 'communities'::regclass \ AND attname IN ('deletion_state', 'deletion_fence_generation', 'deleted_at') \ AND NOT attisdropped ORDER BY attname", ) - .fetch_all(&self.pool) + .fetch_all(&mut *conn) .await?; let column_contract = runtime_columns .iter() @@ -836,7 +876,7 @@ impl DeletionStore { ORDER BY table_name", ) .bind(&required_table_names) - .fetch_all(&self.pool) + .fetch_all(&mut *conn) .await? .into_iter() .collect(); @@ -856,7 +896,7 @@ impl DeletionStore { .copied() .map(str::to_owned) .collect::>(); - let live_fences = self.live_fenced_tables().await?; + let live_fences = live_fenced_tables_on(&mut *conn).await?; let missing_fences = required_fences .difference(&live_fences) .cloned() @@ -882,7 +922,7 @@ impl DeletionStore { AND p.proname = 'enforce_community_tombstone' \ AND NOT t.tgisinternal AND t.tgenabled = 'O')", ) - .fetch_one(&self.pool) + .fetch_one(&mut *conn) .await?; if !required_objects_present { return Err(DbError::DeletionSafety( @@ -2361,7 +2401,12 @@ impl DeletionStore { lease_duration: Duration, ) -> Result { let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); - let mut tx = self.pool.begin().await?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // The assertion owns both the shared ordering lock and the supported // READ COMMITTED check. The lease table is trigger-excluded, so this // explicit admission is its database-enforced write fence. @@ -2425,7 +2470,12 @@ impl DeletionStore { lease_duration: Duration, ) -> Result<()> { let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); - let mut tx = self.pool.begin().await?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; lock_community_deletion_shared(&mut tx, lease.community_id).await?; let lease_until: Option> = sqlx::query_scalar( "UPDATE community_serving_write_leases lease \ @@ -2458,6 +2508,11 @@ impl DeletionStore { /// Release a serving side-effect lease. A stale release is harmless. pub async fn release_serving_write_lease(&self, lease: &ServingWriteLease) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let deleted = sqlx::query( "DELETE FROM community_serving_write_leases \ WHERE id = $1 AND community_id = $2 AND owner = $3 AND generation = $4 \ @@ -2468,7 +2523,7 @@ impl DeletionStore { .bind(&lease.owner) .bind(lease.generation) .bind(lease.fence_generation) - .execute(&self.pool) + .execute(&mut *connection) .await? .rows_affected(); Ok(deleted == 1) @@ -2480,7 +2535,12 @@ impl DeletionStore { /// work remains blocked, preserving an accurate drain without abandoning an /// admitted remote effect. pub async fn verify_serving_write_lease(&self, lease: &ServingWriteLease) -> Result<()> { - let mut tx = self.pool.begin().await?; + let connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; lock_community_deletion_shared(&mut tx, lease.community_id).await?; let valid: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ @@ -2512,6 +2572,11 @@ impl DeletionStore { /// Delete expired serving leases in a bounded batch. pub async fn reap_expired_serving_write_leases(&self, limit: i64) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let affected = sqlx::query( "WITH expired AS ( \ SELECT id FROM community_serving_write_leases \ @@ -2521,7 +2586,7 @@ impl DeletionStore { USING expired WHERE lease.id = expired.id", ) .bind(limit.clamp(1, 10_000)) - .execute(&self.pool) + .execute(&mut *connection) .await? .rows_affected(); Ok(affected) @@ -2529,6 +2594,11 @@ impl DeletionStore { /// Return serving-lease counts and dead-tuple estimate for observability. pub async fn serving_lease_stats(&self) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let row = sqlx::query( "SELECT count(*) FILTER (WHERE lease_until >= now())::BIGINT AS active, \ count(*) FILTER (WHERE lease_until < now())::BIGINT AS expired, \ @@ -2536,7 +2606,7 @@ impl DeletionStore { WHERE relname = 'community_serving_write_leases'), 0) AS dead_tuples \ FROM community_serving_write_leases", ) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await?; Ok(ServingLeaseStats { active: row.try_get("active")?, @@ -2547,12 +2617,17 @@ impl DeletionStore { /// Whether a community remains active and serving-write eligible. pub async fn is_serving_active(&self, community: CommunityId) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 \ AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", ) .bind(community.as_uuid()) - .fetch_one(&self.pool) + .fetch_one(&mut *connection) .await .map_err(Into::into) } @@ -4016,7 +4091,7 @@ mod postgres_tests { .expect("won claim"); let mut open_write = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("open write transaction"); sqlx::query("INSERT INTO pubkey_allowlist (community_id, pubkey) VALUES ($1, $2)") diff --git a/crates/buzz-db/src/store/event.rs b/crates/buzz-db/src/store/event.rs index d685e44485e..d3c0340e07b 100644 --- a/crates/buzz-db/src/store/event.rs +++ b/crates/buzz-db/src/store/event.rs @@ -234,6 +234,26 @@ pub async fn huddle_started_link_exists( ephemeral_channel_id: Uuid, creator_pubkey: &[u8], ) -> Result { + huddle_started_link_exists_with_operation( + pool, + community_id, + parent_channel_id, + ephemeral_channel_id, + creator_pubkey, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +async fn huddle_started_link_exists_with_operation( + pool: &PgPool, + community_id: CommunityId, + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + creator_pubkey: &[u8], + operation: crate::observability::WriterOperation, +) -> Result { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let uuid_needle = format!("%{}%", ephemeral_channel_id); let candidates: Vec = sqlx::query_scalar( r#" @@ -257,7 +277,7 @@ pub async fn huddle_started_link_exists( .bind(HUDDLE_LINK_CONTENT_MAX_BYTES) .bind(uuid_needle) .bind(HUDDLE_LINK_CANDIDATE_LIMIT) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(candidates @@ -274,7 +294,11 @@ pub async fn insert_event( event: &Event, channel_id: Option, ) -> Result<(StoredEvent, bool)> { - let mut connection = pool.acquire().await?; + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; insert_event_on(&mut connection, community_id, event, channel_id).await } @@ -355,7 +379,20 @@ async fn insert_event_on( /// Uses `QueryBuilder` for dynamic filter composition — avoids string concatenation /// while keeping all user values in bind parameters. pub async fn query_events(pool: &PgPool, q: &EventQuery) -> Result> { - let mut conn = pool.acquire().await?; + query_events_with_operation( + pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await +} + +pub(crate) async fn query_events_with_operation( + pool: &PgPool, + q: &EventQuery, + operation: crate::observability::WriterOperation, +) -> Result> { + let mut conn = crate::observability::acquire_writer(pool, operation).await?; query_events_on(&mut conn, q).await } @@ -659,7 +696,11 @@ pub(crate) fn row_to_stored_event(row: sqlx::postgres::PgRow) -> Result Result { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; count_events_on(&mut conn, q).await } @@ -823,12 +864,17 @@ pub async fn soft_delete_event( community_id: CommunityId, event_id: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( "UPDATE events SET deleted_at = NOW() WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", ) .bind(community_id.as_uuid()) .bind(event_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -873,6 +919,11 @@ pub async fn soft_delete_by_coordinate( ) -> Result { let deletion_created_at = DateTime::from_timestamp(deletion_created_at_secs, 0) .ok_or(DbError::InvalidTimestamp(deletion_created_at_secs))?; + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( "UPDATE events SET deleted_at = NOW() \ WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4 AND deleted_at IS NULL \ @@ -883,7 +934,7 @@ pub async fn soft_delete_by_coordinate( .bind(pubkey) .bind(d_tag) .bind(deletion_created_at) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -901,7 +952,12 @@ pub async fn soft_delete_event_and_update_thread( parent_event_id: Option<&[u8]>, root_event_id: Option<&[u8]>, ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let result = sqlx::query( "UPDATE events SET deleted_at = NOW() WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL", @@ -949,6 +1005,11 @@ pub async fn get_last_message_at( community_id: CommunityId, channel_id: uuid::Uuid, ) -> Result>> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; let row = sqlx::query( "SELECT created_at FROM events \ WHERE community_id = $1 AND channel_id = $2 AND deleted_at IS NULL \ @@ -956,7 +1017,7 @@ pub async fn get_last_message_at( ) .bind(community_id.as_uuid()) .bind(channel_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -977,6 +1038,11 @@ pub async fn get_last_message_at_bulk( if channel_ids.is_empty() { return Ok(std::collections::HashMap::new()); } + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; let mut qb: QueryBuilder = QueryBuilder::new( "SELECT channel_id, MAX(created_at) as last_at FROM events \ @@ -990,7 +1056,7 @@ pub async fn get_last_message_at_bulk( } qb.push(") GROUP BY channel_id"); - let rows = qb.build().fetch_all(pool).await?; + let rows = qb.build().fetch_all(&mut *connection).await?; let mut map = std::collections::HashMap::with_capacity(rows.len()); for row in rows { @@ -1011,13 +1077,29 @@ pub async fn get_event_by_id( community_id: CommunityId, id_bytes: &[u8], ) -> Result> { + get_event_by_id_with_operation( + pool, + community_id, + id_bytes, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +pub(crate) async fn get_event_by_id_with_operation( + pool: &PgPool, + community_id: CommunityId, + id_bytes: &[u8], + operation: crate::observability::WriterOperation, +) -> Result> { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let row = sqlx::query( "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id \ FROM events WHERE community_id = $1 AND id = $2 AND deleted_at IS NULL ORDER BY created_at DESC LIMIT 1", ) .bind(community_id.as_uuid()) .bind(id_bytes) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -1038,6 +1120,11 @@ pub async fn get_latest_global_replaceable( kind: i32, pubkey_bytes: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id \ FROM events \ @@ -1048,7 +1135,7 @@ pub async fn get_latest_global_replaceable( .bind(community_id.as_uuid()) .bind(kind) .bind(pubkey_bytes) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -1067,13 +1154,29 @@ pub async fn get_event_by_id_including_deleted( community_id: CommunityId, id_bytes: &[u8], ) -> Result> { + get_event_by_id_including_deleted_with_operation( + pool, + community_id, + id_bytes, + crate::observability::WriterOperation::Authorization, + ) + .await +} + +pub(crate) async fn get_event_by_id_including_deleted_with_operation( + pool: &PgPool, + community_id: CommunityId, + id_bytes: &[u8], + operation: crate::observability::WriterOperation, +) -> Result> { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let row = sqlx::query( "SELECT id, pubkey, created_at, kind, tags, content, sig, received_at, channel_id \ FROM events WHERE community_id = $1 AND id = $2 ORDER BY created_at DESC LIMIT 1", ) .bind(community_id.as_uuid()) .bind(id_bytes) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -1090,11 +1193,26 @@ pub async fn get_events_by_ids( pool: &PgPool, community_id: CommunityId, ids: &[&[u8]], +) -> Result> { + get_events_by_ids_with_operation( + pool, + community_id, + ids, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await +} + +pub(crate) async fn get_events_by_ids_with_operation( + pool: &PgPool, + community_id: CommunityId, + ids: &[&[u8]], + operation: crate::observability::WriterOperation, ) -> Result> { if ids.is_empty() { return Ok(vec![]); } - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer(pool, operation).await?; get_events_by_ids_on(&mut conn, community_id, ids).await } @@ -1340,7 +1458,12 @@ pub async fn insert_event_with_thread_metadata( channel_id: Option, thread_meta: Option>, ) -> Result<(StoredEvent, bool)> { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let result = insert_event_with_thread_metadata_tx(&mut tx, community_id, event, channel_id, thread_meta) .await?; @@ -1378,7 +1501,46 @@ impl Db { /// explicit, per-callsite decision, never a change to this method. #[datastore_span(name = "query_events", system = "postgresql")] pub async fn query_events(&self, q: &EventQuery) -> Result> { - crate::event::query_events(&self.pool, q).await + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::Authorization, + ) + .await + } + + /// Query authoritative event state that directly controls a durable event + /// mutation or its post-commit side effects. + #[datastore_span(name = "query_events_for_event_write", system = "postgresql")] + pub async fn query_events_for_event_write(&self, q: &EventQuery) -> Result> { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + + /// Query authoritative event state for startup reconciliation. + #[datastore_span(name = "query_events_for_bootstrap", system = "postgresql")] + pub async fn query_events_for_bootstrap(&self, q: &EventQuery) -> Result> { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::Bootstrap, + ) + .await + } + + /// Query authoritative event state for background reconciliation or repair. + #[datastore_span(name = "query_events_for_maintenance", system = "postgresql")] + pub async fn query_events_for_maintenance(&self, q: &EventQuery) -> Result> { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::Maintenance, + ) + .await } /// [`Db::query_events`] with replica routing — the opt-in fast path for @@ -1404,7 +1566,14 @@ impl Db { q: &EventQuery, ) -> Result> { let predicate = crate::RoutePredicate::for_query(q, self.replica_read_max_age.is_some()); - match self.route_read(path, predicate).await { + match self + .route_read( + path, + predicate, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { crate::RouteDecision::Replica(mut tx, _entry, reason) => { match crate::event::query_events_on(&mut tx, q).await { Ok(events) => { @@ -1416,11 +1585,23 @@ impl Db { // writer rather than surfacing a routed error. tracing::warn!(path, "replica read failed; re-running on writer: {e}"); Self::record_route(path, "writer", "replica_error"); - crate::event::query_events(&self.pool, q).await + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await } } } - crate::RouteDecision::Writer => crate::event::query_events(&self.pool, q).await, + crate::RouteDecision::Writer => { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await + } } } @@ -1438,7 +1619,14 @@ impl Db { path: &'static str, q: &EventQuery, ) -> Result> { - match self.route_read(path, crate::RoutePredicate::Bounded).await { + match self + .route_read( + path, + crate::RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { crate::RouteDecision::Replica(mut tx, _entry, reason) => { match crate::event::query_events_on(&mut tx, q).await { Ok(events) => { @@ -1448,11 +1636,23 @@ impl Db { Err(e) => { tracing::warn!(path, "replica read failed; re-running on writer: {e}"); Self::record_route(path, "writer", "replica_error"); - crate::event::query_events(&self.pool, q).await + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await } } } - crate::RouteDecision::Writer => crate::event::query_events(&self.pool, q).await, + crate::RouteDecision::Writer => { + crate::event::query_events_with_operation( + &self.pool, + q, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await + } } } @@ -1477,7 +1677,14 @@ impl Db { /// the error to the accepted budget `B`. #[datastore_span(name = "count_events_routed", system = "postgresql")] pub async fn count_events_routed(&self, path: &'static str, q: &EventQuery) -> Result { - match self.route_read(path, crate::RoutePredicate::Bounded).await { + match self + .route_read( + path, + crate::RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { crate::RouteDecision::Replica(mut tx, _entry, reason) => { match crate::event::count_events_on(&mut tx, q).await { Ok(count) => { @@ -1515,6 +1722,29 @@ impl Db { .await } + /// Validate a huddle link while admitting a huddle event for persistence. + #[datastore_span( + name = "huddle_started_link_exists_for_event_write", + system = "postgresql" + )] + pub async fn huddle_started_link_exists_for_event_write( + &self, + community_id: CommunityId, + parent_channel_id: Uuid, + ephemeral_channel_id: Uuid, + creator_pubkey: &[u8], + ) -> Result { + crate::event::huddle_started_link_exists_with_operation( + &self.pool, + community_id, + parent_channel_id, + ephemeral_channel_id, + creator_pubkey, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Fetch the latest replaceable event for a (kind, pubkey) pair. /// /// Uses canonical NIP-16 ordering: `created_at DESC, id ASC`. @@ -1543,6 +1773,23 @@ impl Db { crate::event::get_event_by_id(&self.pool, community_id, id_bytes).await } + /// Fetch an event as a prerequisite of an event write or durable + /// post-write side effect. + #[datastore_span(name = "get_event_by_id_for_event_write", system = "postgresql")] + pub async fn get_event_by_id_for_event_write( + &self, + community_id: CommunityId, + id_bytes: &[u8], + ) -> Result> { + crate::event::get_event_by_id_with_operation( + &self.pool, + community_id, + id_bytes, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Fetches a single event by its raw ID bytes, **including soft-deleted rows**. #[datastore_span(name = "get_event_by_id_including_deleted", system = "postgresql")] pub async fn get_event_by_id_including_deleted( @@ -1553,6 +1800,26 @@ impl Db { crate::event::get_event_by_id_including_deleted(&self.pool, community_id, id_bytes).await } + /// Fetch an event including tombstones as a prerequisite of an event + /// write or durable post-write side effect. + #[datastore_span( + name = "get_event_by_id_including_deleted_for_event_write", + system = "postgresql" + )] + pub async fn get_event_by_id_including_deleted_for_event_write( + &self, + community_id: CommunityId, + id_bytes: &[u8], + ) -> Result> { + crate::event::get_event_by_id_including_deleted_with_operation( + &self.pool, + community_id, + id_bytes, + crate::observability::WriterOperation::EventWrite, + ) + .await + } + /// Soft-deletes an event. Returns `Ok(true)` if deleted, `Ok(false)` if already deleted. #[datastore_span(name = "soft_delete_event", system = "postgresql")] pub async fn soft_delete_event( @@ -1633,7 +1900,13 @@ impl Db { community_id: CommunityId, ids: &[&[u8]], ) -> Result> { - crate::event::get_events_by_ids(&self.pool, community_id, ids).await + crate::event::get_events_by_ids_with_operation( + &self.pool, + community_id, + ids, + crate::observability::WriterOperation::Authorization, + ) + .await } /// [`Db::get_events_by_ids`] with replica routing — same contract and @@ -1650,7 +1923,14 @@ impl Db { community_id: CommunityId, ids: &[&[u8]], ) -> Result> { - match self.route_read(path, crate::RoutePredicate::Bounded).await { + match self + .route_read( + path, + crate::RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { crate::RouteDecision::Replica(mut tx, _entry, reason) => { match crate::event::get_events_by_ids_on(&mut tx, community_id, ids).await { Ok(events) => { @@ -1660,12 +1940,24 @@ impl Db { Err(e) => { tracing::warn!(path, "replica read failed; re-running on writer: {e}"); Self::record_route(path, "writer", "replica_error"); - crate::event::get_events_by_ids(&self.pool, community_id, ids).await + crate::event::get_events_by_ids_with_operation( + &self.pool, + community_id, + ids, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await } } } crate::RouteDecision::Writer => { - crate::event::get_events_by_ids(&self.pool, community_id, ids).await + crate::event::get_events_by_ids_with_operation( + &self.pool, + community_id, + ids, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await } } } @@ -1703,6 +1995,11 @@ impl Db { /// Runs a single UPDATE touching only NIP-33 rows with NULL d_tag. #[datastore_span(name = "backfill_d_tags", system = "postgresql")] pub async fn backfill_d_tags(&self) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; let result = sqlx::query( "UPDATE events \ SET d_tag = COALESCE( \ @@ -1713,7 +2010,7 @@ impl Db { WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL \ AND community_write_allowed(community_id)", ) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) } @@ -1726,6 +2023,11 @@ impl Db { channel_id: Uuid, relay_pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + &self.pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( "UPDATE events SET deleted_at = NOW() \ WHERE community_id = $1 AND channel_id = $2 AND pubkey = $3 AND deleted_at IS NULL AND kind IN (39000, 39001, 39002)", @@ -1733,7 +2035,7 @@ impl Db { .bind(community_id.as_uuid()) .bind(channel_id) .bind(relay_pubkey) - .execute(&self.pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) } diff --git a/crates/buzz-db/src/store/feed.rs b/crates/buzz-db/src/store/feed.rs index 7047bbdd9b7..5819136fe6a 100644 --- a/crates/buzz-db/src/store/feed.rs +++ b/crates/buzz-db/src/store/feed.rs @@ -134,7 +134,11 @@ pub async fn query_mentions( since: Option>, limit: i64, ) -> Result> { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; query_mentions_on( &mut conn, community, @@ -218,7 +222,11 @@ pub async fn query_needs_action( since: Option>, limit: i64, ) -> Result> { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; query_needs_action_on( &mut conn, community, @@ -287,7 +295,11 @@ pub async fn query_activity( since: Option>, limit: i64, ) -> Result> { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; query_activity_on(&mut conn, community, accessible_channel_ids, since, limit).await } @@ -345,7 +357,14 @@ impl Db { since: Option>, limit: i64, ) -> Result> { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => match crate::feed::query_mentions_on( &mut tx, community, @@ -422,7 +441,14 @@ impl Db { since: Option>, limit: i64, ) -> Result> { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => { match crate::feed::query_needs_action_on( &mut tx, @@ -492,7 +518,14 @@ impl Db { since: Option>, limit: i64, ) -> Result> { - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => match crate::feed::query_activity_on( &mut tx, community, diff --git a/crates/buzz-db/src/store/git_repo.rs b/crates/buzz-db/src/store/git_repo.rs index bc4e70e151e..fac10c3f610 100644 --- a/crates/buzz-db/src/store/git_repo.rs +++ b/crates/buzz-db/src/store/git_repo.rs @@ -50,13 +50,18 @@ pub async fn repo_name_owner( community: CommunityId, repo_id: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT owner_pubkey FROM git_repo_names \ WHERE community_id = $1 AND repo_id = $2", ) .bind(community.as_uuid()) .bind(repo_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(|r| r.try_get("owner_pubkey")) .transpose() @@ -85,6 +90,11 @@ pub async fn reserve_repo_name( repo_id: &str, owner_pubkey: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; // Atomic claim: insert only if the (community, repo) is free. RETURNING is // non-empty exactly when *this* statement inserted the row, so it cleanly // distinguishes "I claimed it" from "someone already holds it" without a @@ -98,7 +108,7 @@ pub async fn reserve_repo_name( .bind(community.as_uuid()) .bind(repo_id) .bind(owner_pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; if inserted.is_some() { @@ -113,7 +123,7 @@ pub async fn reserve_repo_name( ) .bind(community.as_uuid()) .bind(repo_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match existing { @@ -145,13 +155,18 @@ pub async fn count_repos_for_owner( community: CommunityId, owner_pubkey: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT COUNT(*) AS n FROM git_repo_names \ WHERE community_id = $1 AND owner_pubkey = $2", ) .bind(community.as_uuid()) .bind(owner_pubkey) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; row.try_get("n").map_err(crate::error::DbError::from) } @@ -168,6 +183,11 @@ pub async fn release_repo_name( repo_id: &str, owner_pubkey: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let result = sqlx::query( "DELETE FROM git_repo_names \ WHERE community_id = $1 AND repo_id = $2 AND owner_pubkey = $3", @@ -175,7 +195,7 @@ pub async fn release_repo_name( .bind(community.as_uuid()) .bind(repo_id) .bind(owner_pubkey) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) } diff --git a/crates/buzz-db/src/store/moderation.rs b/crates/buzz-db/src/store/moderation.rs index 94e550185d5..b5cc4d30930 100644 --- a/crates/buzz-db/src/store/moderation.rs +++ b/crates/buzz-db/src/store/moderation.rs @@ -464,6 +464,11 @@ pub async fn restriction_state( community: CommunityId, pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( r#" SELECT @@ -475,7 +480,7 @@ pub async fn restriction_state( ) .bind(community.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { diff --git a/crates/buzz-db/src/store/partition.rs b/crates/buzz-db/src/store/partition.rs index ba252f71f4a..179ba60b782 100644 --- a/crates/buzz-db/src/store/partition.rs +++ b/crates/buzz-db/src/store/partition.rs @@ -16,6 +16,11 @@ const PARTITIONED_TABLES: &[&str] = &["events", "delivery_log"]; /// Ensures monthly partition tables exist for the next `months_ahead` months. pub async fn ensure_future_partitions(pool: &PgPool, months_ahead: u32) -> Result<()> { let now = Utc::now(); + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Bootstrap, + ) + .await?; for i in 0..=(months_ahead as i32) { let year = now.year(); @@ -50,7 +55,7 @@ pub async fn ensure_future_partitions(pool: &PgPool, months_ahead: u32) -> Resul let end_str = end.format("%Y-%m-%d").to_string(); for table in PARTITIONED_TABLES { - ensure_partition(pool, table, &start_str, &end_str, &suffix).await?; + ensure_partition(&mut connection, table, &start_str, &end_str, &suffix).await?; } } @@ -82,7 +87,7 @@ fn validate_date_str(s: &str) -> bool { } async fn ensure_partition( - pool: &PgPool, + connection: &mut sqlx::PgConnection, table_name: &str, start_date_str: &str, end_date_str: &str, @@ -123,7 +128,7 @@ async fn ensure_partition( "#, ) .bind(&partition_name) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; let cnt: i64 = row.try_get("cnt")?; @@ -137,7 +142,10 @@ async fn ensure_partition( FOR VALUES FROM ('{start_date_str}') TO ('{end_date_str}')" ); - match sqlx::query(sqlx::AssertSqlSafe(sql)).execute(pool).await { + match sqlx::query(sqlx::AssertSqlSafe(sql)) + .execute(&mut *connection) + .await + { Ok(_) => { info!("added partition {partition_name}"); Ok(()) diff --git a/crates/buzz-db/src/store/push.rs b/crates/buzz-db/src/store/push.rs index 710ffb931c6..59c44fc5a83 100644 --- a/crates/buzz-db/src/store/push.rs +++ b/crates/buzz-db/src/store/push.rs @@ -14,6 +14,21 @@ use crate::error::Result; use crate::Db; use buzz_datastore_tracing::datastore_span; +async fn acquire_operation_connection( + pool: &PgPool, + operation: crate::observability::WriterOperation, +) -> Result> { + Ok(crate::observability::acquire_writer(pool, operation).await?) +} + +async fn begin_operation_transaction( + pool: &PgPool, + operation: crate::observability::WriterOperation, +) -> Result> { + let connection = acquire_operation_connection(pool, operation).await?; + Ok(sqlx::Transaction::begin(connection, None).await?) +} + /// Namespace for the per-community push-gate advisory lock. Must match the /// key built inside the `enqueue_push_match_job` trigger (migration 0023): /// event inserts take it SHARED there; every lease transition that can make @@ -498,7 +513,9 @@ async fn replace_lease( // lease" to "eligible"; serialize it against the trigger's shared gate // lock (gate → lease row, matching accept_lease_event's global order). // Revocations (is_active = false) never make eligibility true and skip it. - let mut tx = pool.begin().await?; + let mut tx = + begin_operation_transaction(pool, crate::observability::WriterOperation::EventWrite) + .await?; if is_active { acquire_push_gate_lock(&mut tx, community).await?; } @@ -651,7 +668,9 @@ pub async fn enqueue_wakes( if requests.is_empty() { return Ok(Vec::new()); } - let mut tx = pool.begin().await?; + let mut tx = + begin_operation_transaction(pool, crate::observability::WriterOperation::Maintenance) + .await?; // 1. Lock and read the current lease row for every distinct requested // (author, installation), in deterministic order. @@ -854,7 +873,13 @@ pub async fn claim_due_match_batch( lease_until, |pool, community, ids| async move { let refs: Vec<&[u8]> = ids.iter().map(Vec::as_slice).collect(); - crate::event::get_events_by_ids(&pool, community, &refs).await + crate::event::get_events_by_ids_with_operation( + &pool, + community, + &refs, + crate::observability::WriterOperation::Maintenance, + ) + .await }, ) .await @@ -871,6 +896,9 @@ where Fut: std::future::Future>>, { let claim_id = Uuid::new_v4(); + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let rows = sqlx::query( r#" WITH target AS ( @@ -905,11 +933,16 @@ where .bind(lease_until) .bind(MAX_MATCH_ATTEMPTS) .bind(limit) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; if rows.is_empty() { return Ok(None); } + // The claim query is a single autocommitted statement. Release its pool + // slot before loading source events, because the production loader owns a + // separately attributed acquisition. Holding this connection across the + // load would self-starve a supported size-one writer pool. + drop(connection); let community = CommunityId::from_uuid(rows[0].try_get("community_id")?); let mut attempts = std::collections::HashMap::with_capacity(rows.len()); for row in &rows { @@ -932,6 +965,9 @@ where // recoverable after their claim lease expires. let gone: Vec> = attempts.into_keys().collect(); if !gone.is_empty() { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; sqlx::query( "DELETE FROM push_match_queue \ WHERE community_id=$1 AND claim_id=$2 AND state='matching' AND event_id = ANY($3)", @@ -939,7 +975,7 @@ where .bind(community.as_uuid()) .bind(claim_id) .bind(&gone) - .execute(pool) + .execute(&mut *connection) .await?; } if jobs.is_empty() { @@ -959,26 +995,32 @@ where /// served by the due partial index, so putting it in every claim made claims /// slower exactly when a backlog needed them fastest. pub async fn reap_exhausted_matches(pool: &PgPool) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; Ok(sqlx::query( "DELETE FROM push_match_queue WHERE attempts >= $1 \ AND (state='pending' OR (state='matching' AND lease_until < now())) \ AND community_write_allowed(community_id)", ) .bind(MAX_MATCH_ATTEMPTS) - .execute(pool) + .execute(&mut *connection) .await? .rows_affected()) } /// Load active endpoint-enabled leases for one tenant. pub async fn active_match_leases(pool: &PgPool, community: CommunityId) -> Result> { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let rows = sqlx::query( "SELECT author, installation_id, generation, subscriptions, expires_at \ FROM push_leases WHERE community_id=$1 AND active AND endpoint_enabled \ AND expires_at > EXTRACT(EPOCH FROM now())::bigint", ) .bind(community.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() .map(|row| { @@ -1005,6 +1047,9 @@ pub async fn complete_match_batch( if event_ids.is_empty() { return Ok(0); } + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; Ok(sqlx::query( "DELETE FROM push_match_queue \ WHERE community_id=$1 AND claim_id=$2 AND state='matching' AND event_id = ANY($3)", @@ -1012,7 +1057,7 @@ pub async fn complete_match_batch( .bind(community.as_uuid()) .bind(claim_id) .bind(event_ids) - .execute(pool) + .execute(&mut *connection) .await? .rows_affected()) } @@ -1029,6 +1074,9 @@ pub async fn retry_match_batch( if event_ids.is_empty() { return Ok(0); } + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; Ok(sqlx::query( "UPDATE push_match_queue \ SET state='pending', claim_id=NULL, lease_until=NULL, next_attempt_at=$4 \ @@ -1038,7 +1086,7 @@ pub async fn retry_match_batch( .bind(claim_id) .bind(event_ids) .bind(next) - .execute(pool) + .execute(&mut *connection) .await? .rows_affected()) } @@ -1054,6 +1102,9 @@ pub async fn claim_due_wakes( lease_until: DateTime, ) -> Result> { let claim_id = Uuid::new_v4(); + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let rows = sqlx::query( r#" WITH candidates AS ( @@ -1101,7 +1152,7 @@ pub async fn claim_due_wakes( .bind(limit) .bind(claim_id) .bind(lease_until) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter().map(row_to_claimed_wake).collect() @@ -1118,6 +1169,9 @@ pub async fn revalidate_wake_for_send( id: Uuid, claim_id: Uuid, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let row = sqlx::query( r#" SELECT o.community_id, o.id, o.claim_id, o.event_id, e.channel_id, @@ -1149,7 +1203,7 @@ pub async fn revalidate_wake_for_send( .bind(community.as_uuid()) .bind(id) .bind(claim_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(row_to_claimed_wake) @@ -1166,6 +1220,9 @@ pub async fn complete_wake( id: Uuid, claim_id: Uuid, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "UPDATE push_wake_outbox \ SET state = 'delivered', claim_id = NULL, lease_until = NULL \ @@ -1174,7 +1231,7 @@ pub async fn complete_wake( .bind(community.as_uuid()) .bind(id) .bind(claim_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -1187,6 +1244,9 @@ pub async fn retry_wake( claim_id: Uuid, next_attempt_at: DateTime, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "UPDATE push_wake_outbox \ SET state = 'pending', next_attempt_at = $4, claim_id = NULL, lease_until = NULL \ @@ -1196,7 +1256,7 @@ pub async fn retry_wake( .bind(id) .bind(claim_id) .bind(next_attempt_at) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -1208,6 +1268,9 @@ pub async fn fail_wake( id: Uuid, claim_id: Uuid, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "UPDATE push_wake_outbox \ SET state = 'failed', claim_id = NULL, lease_until = NULL \ @@ -1216,7 +1279,7 @@ pub async fn fail_wake( .bind(community.as_uuid()) .bind(id) .bind(claim_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -1232,6 +1295,9 @@ pub async fn disable_endpoint_generation( installation_id: &str, generation: i64, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "UPDATE push_leases SET endpoint_enabled = false, updated_at = now() \ WHERE community_id = $1 AND author = $2 AND installation_id = $3 \ @@ -1241,7 +1307,7 @@ pub async fn disable_endpoint_generation( .bind(author) .bind(installation_id) .bind(generation) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -1256,6 +1322,9 @@ pub async fn prune_wake_outbox( community: CommunityId, before: DateTime, ) -> Result { + let mut connection = + acquire_operation_connection(pool, crate::observability::WriterOperation::Maintenance) + .await?; let result = sqlx::query( "DELETE FROM push_wake_outbox o \ WHERE o.community_id = $1 AND o.created_at < $2 \ @@ -1268,7 +1337,7 @@ pub async fn prune_wake_outbox( ) .bind(community.as_uuid()) .bind(before) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) } @@ -1466,7 +1535,9 @@ impl Db { mod postgres_tests { use super::*; use crate::migration; + use sqlx::postgres::PgPoolOptions; use std::sync::Arc; + use std::time::Duration; use tokio::sync::Barrier; async fn setup_pool() -> PgPool { @@ -2160,6 +2231,51 @@ mod postgres_tests { ); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn matcher_claim_and_load_support_size_one_pool() { + let setup = setup_pool().await; + sqlx::query("DELETE FROM push_match_queue") + .execute(&setup) + .await + .expect("drain matcher queue"); + let community = make_community(&setup).await; + activate(&setup, community, &[83; 32], "install", &[84; 32], 1).await; + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "size one") + .sign_with_keys(&nostr::Keys::generate()) + .expect("sign event"); + crate::event::insert_event(&setup, community, &event, None) + .await + .expect("insert event"); + setup.close().await; + + let pool = PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_millis(250)) + .connect(&crate::test_support::database_url()) + .await + .expect("connect size-one matcher pool"); + let batch = tokio::time::timeout( + Duration::from_secs(2), + claim_due_match_batch(&pool, 16, Utc::now() + chrono::Duration::minutes(1)), + ) + .await + .expect("matcher must not self-starve on a size-one pool") + .expect("claim and source load must succeed") + .expect("seeded matcher job must be claimed"); + assert_eq!(batch.community, community); + assert_eq!(batch.jobs.len(), 1); + assert_eq!(batch.jobs[0].event.event.id, event.id); + + let ids = vec![event.id.as_bytes().to_vec()]; + assert_eq!( + complete_match_batch(&pool, community, batch.claim_id, &ids) + .await + .expect("complete size-one matcher batch"), + 1 + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn matcher_claim_is_exclusive_across_workers() { diff --git a/crates/buzz-db/src/store/reaction.rs b/crates/buzz-db/src/store/reaction.rs index 6494d856639..1c4ee19ead7 100644 --- a/crates/buzz-db/src/store/reaction.rs +++ b/crates/buzz-db/src/store/reaction.rs @@ -111,6 +111,11 @@ pub async fn add_reaction( emoji: &str, reaction_event_id: Option<&[u8]>, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query(ADD_REACTION_SQL) .bind(community.as_uuid()) .bind(event_created_at) @@ -118,7 +123,7 @@ pub async fn add_reaction( .bind(pubkey) .bind(emoji) .bind(reaction_event_id) - .execute(pool) + .execute(&mut *connection) .await?; // Three cases: @@ -173,7 +178,12 @@ pub async fn insert_reaction_event_with_thread_metadata( actor_pubkey: &[u8], emoji: &str, ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let target_row = sqlx::query( "SELECT created_at FROM events \ @@ -236,6 +246,11 @@ pub async fn remove_reaction( pubkey: &[u8], emoji: &str, ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( r#" UPDATE reactions @@ -253,7 +268,7 @@ pub async fn remove_reaction( .bind(event_id) .bind(pubkey) .bind(emoji) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -267,6 +282,11 @@ pub async fn remove_reaction_by_source_event_id( community: CommunityId, reaction_event_id: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( r#" UPDATE reactions @@ -278,7 +298,7 @@ pub async fn remove_reaction_by_source_event_id( ) .bind(community.as_uuid()) .bind(reaction_event_id) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -293,6 +313,11 @@ pub async fn get_active_reaction_record( pubkey: &[u8], emoji: &str, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let row = sqlx::query( r#" SELECT reaction_event_id @@ -311,7 +336,7 @@ pub async fn get_active_reaction_record( .bind(event_created_at) .bind(pubkey) .bind(emoji) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(|row| -> Result { @@ -335,6 +360,11 @@ pub async fn set_reaction_event_id( emoji: &str, reaction_event_id: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let result = sqlx::query( r#" UPDATE reactions @@ -353,7 +383,7 @@ pub async fn set_reaction_event_id( .bind(event_id) .bind(pubkey) .bind(emoji) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) @@ -376,6 +406,11 @@ pub async fn get_reactions( limit: u32, _cursor: Option<&str>, ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; // Two-step query: first get the limited set of distinct emoji groups, // then fetch all rows for those groups. This ensures `limit` applies to // emoji groups (the API contract), not raw rows — so one busy emoji @@ -405,7 +440,7 @@ pub async fn get_reactions( .bind(event_id) .bind(event_created_at) .bind(limit as i64) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; // Group individual rows by emoji in Rust. @@ -466,6 +501,11 @@ pub async fn get_reactions_bulk( // Run one query per event. For typical message-list sizes (<=100 events) // this is acceptable; a single-query approach with dynamic IN clauses over // composite keys can be added later if needed. + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; let mut entries = Vec::new(); for (event_id, event_created_at) in event_ids { @@ -484,7 +524,7 @@ pub async fn get_reactions_bulk( .bind(community.as_uuid()) .bind(*event_id) .bind(event_created_at) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; if rows.is_empty() { diff --git a/crates/buzz-db/src/store/relay_invite.rs b/crates/buzz-db/src/store/relay_invite.rs index 6c90c31b944..4bb48e121ca 100644 --- a/crates/buzz-db/src/store/relay_invite.rs +++ b/crates/buzz-db/src/store/relay_invite.rs @@ -116,7 +116,12 @@ pub async fn mint_relay_invite( // community-scoped database write. The trigger remains the final backstop, // but this typed guard keeps a quiescing community from surfacing as an // opaque SQLSTATE/HTTP 500 at the API boundary. - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; crate::deletion::DeletionStore::new(pool.clone()) .guard_transaction(&mut tx, community) .await?; @@ -172,6 +177,11 @@ const RETENTION_SWEEP_BATCH_SIZE: i64 = 1_000; /// expiry index makes old rows drain first without turning cleanup into an /// unbounded transaction. pub async fn reap_expired_relay_invites(pool: &PgPool, cutoff: DateTime) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Maintenance, + ) + .await?; let result = sqlx::query( "DELETE FROM relay_invites \ WHERE (community_id, id) IN (\ @@ -184,7 +194,7 @@ pub async fn reap_expired_relay_invites(pool: &PgPool, cutoff: DateTime) -> ) .bind(cutoff) .bind(RETENTION_SWEEP_BATCH_SIZE) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) @@ -216,7 +226,12 @@ pub async fn claim_relay_invite( claimer_pubkey: &str, policy_version: Option<&str>, ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // 2. SELECT FOR UPDATE — lock the invite row for the duration of this txn. let row = sqlx::query( diff --git a/crates/buzz-db/src/store/relay_members.rs b/crates/buzz-db/src/store/relay_members.rs index 9a5b6f91a24..ecde3924fef 100644 --- a/crates/buzz-db/src/store/relay_members.rs +++ b/crates/buzz-db/src/store/relay_members.rs @@ -32,7 +32,8 @@ pub struct RelayMember { /// Returns `true` if `pubkey` (64-char hex) is a member of `community`. pub async fn is_relay_member(pool: &PgPool, community: CommunityId, pubkey: &str) -> Result { - let mut conn = pool.acquire().await?; + let mut conn = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; is_relay_member_on(&mut conn, community, pubkey).await } @@ -57,12 +58,14 @@ pub(crate) async fn is_relay_member_on( /// (`bootstrap_owner`) and operator provisioning still populate it — this is /// how the workspace-profile gate detects whether a steward exists. pub async fn has_admin_or_owner(pool: &PgPool, community: CommunityId) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let row = sqlx::query( "SELECT 1 FROM relay_members \ WHERE community_id = $1 AND role IN ('admin', 'owner') LIMIT 1", ) .bind(community.as_uuid()) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.is_some()) } @@ -73,13 +76,15 @@ pub async fn get_relay_member( community: CommunityId, pubkey: &str, ) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let row = sqlx::query( "SELECT pubkey, role, added_by, created_at, updated_at \ FROM relay_members WHERE community_id = $1 AND pubkey = $2", ) .bind(community.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(|r| -> std::result::Result { @@ -97,12 +102,26 @@ pub async fn get_relay_member( /// Returns all relay members of `community` ordered by `created_at` ascending. pub async fn list_relay_members(pool: &PgPool, community: CommunityId) -> Result> { + list_relay_members_with_operation( + pool, + community, + observability::WriterOperation::Authorization, + ) + .await +} + +async fn list_relay_members_with_operation( + pool: &PgPool, + community: CommunityId, + operation: observability::WriterOperation, +) -> Result> { + let mut connection = observability::acquire_writer(pool, operation).await?; let rows = sqlx::query( "SELECT pubkey, role, added_by, created_at, updated_at \ FROM relay_members WHERE community_id = $1 ORDER BY created_at ASC", ) .bind(community.as_uuid()) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() @@ -131,6 +150,8 @@ pub async fn add_relay_member( role: &str, added_by: Option<&str>, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let result = sqlx::query( "INSERT INTO relay_members (community_id, pubkey, role, added_by) \ VALUES ($1, $2, $3, $4) ON CONFLICT (community_id, pubkey) DO NOTHING", @@ -139,7 +160,7 @@ pub async fn add_relay_member( .bind(pubkey) .bind(role) .bind(added_by) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) } @@ -156,7 +177,9 @@ pub async fn claim_relay_membership( role: &str, policy_version: Option<&str>, ) -> Result { - let mut tx = pool.begin().await?; + let connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; let inserted = sqlx::query( "INSERT INTO relay_members (community_id, pubkey, role, added_by) \ VALUES ($1, $2, $3, 'invite') \ @@ -193,6 +216,8 @@ pub async fn has_join_policy_acceptance( pubkey: &str, policy_version: &str, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let row = sqlx::query( "SELECT 1 FROM join_policy_acceptances \ WHERE community_id = $1 AND pubkey = $2 AND policy_version = $3", @@ -200,7 +225,7 @@ pub async fn has_join_policy_acceptance( .bind(community.as_uuid()) .bind(pubkey) .bind(policy_version) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.is_some()) } @@ -228,13 +253,15 @@ pub async fn remove_relay_member( community: CommunityId, pubkey: &str, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let result = sqlx::query( "DELETE FROM relay_members \ WHERE community_id = $1 AND pubkey = $2 AND role <> 'owner'", ) .bind(community.as_uuid()) .bind(pubkey) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() > 0 { @@ -246,7 +273,7 @@ pub async fn remove_relay_member( let exists = sqlx::query("SELECT 1 FROM relay_members WHERE community_id = $1 AND pubkey = $2") .bind(community.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; if exists.is_some() { @@ -275,13 +302,15 @@ pub async fn remove_relay_member_if_role( pubkey: &str, expected_role: &str, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let result = sqlx::query( "DELETE FROM relay_members WHERE community_id = $1 AND pubkey = $2 AND role = $3", ) .bind(community.as_uuid()) .bind(pubkey) .bind(expected_role) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() > 0 { @@ -293,7 +322,7 @@ pub async fn remove_relay_member_if_role( let row = sqlx::query("SELECT role FROM relay_members WHERE community_id = $1 AND pubkey = $2") .bind(community.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; match row { @@ -320,6 +349,8 @@ pub async fn update_relay_member_role( pubkey: &str, new_role: &str, ) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; let result = sqlx::query( "UPDATE relay_members SET role = $1, updated_at = now() \ WHERE community_id = $2 AND pubkey = $3 AND role <> 'owner'", @@ -327,7 +358,7 @@ pub async fn update_relay_member_role( .bind(new_role) .bind(community.as_uuid()) .bind(pubkey) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() > 0) } @@ -351,9 +382,25 @@ pub async fn bootstrap_owner( pool: &PgPool, community: CommunityId, owner_pubkey: &str, +) -> Result<()> { + bootstrap_owner_with_operation( + pool, + community, + owner_pubkey, + observability::WriterOperation::Bootstrap, + ) + .await +} + +async fn bootstrap_owner_with_operation( + pool: &PgPool, + community: CommunityId, + owner_pubkey: &str, + operation: observability::WriterOperation, ) -> Result<()> { let pubkey = owner_pubkey.to_ascii_lowercase(); - let mut tx = pool.begin().await?; + let connection = observability::acquire_writer(pool, operation).await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // 1. Upsert the configured owner for this community. sqlx::query( @@ -472,7 +519,9 @@ pub async fn transfer_ownership( ) -> Result { let pubkey = new_owner_pubkey.to_ascii_lowercase(); let expected_owner = expected_owner_pubkey.to_ascii_lowercase(); - let mut tx = pool.begin().await?; + let connection = + observability::acquire_writer(pool, observability::WriterOperation::Authorization).await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // 1. Serialize on the transferee so concurrent transfers to the same // recipient cannot both pass the ownership count check. @@ -573,12 +622,14 @@ pub async fn transfer_ownership( /// The empty-table guard prevents re-adding members that were intentionally /// removed by an admin after the initial backfill. pub async fn backfill_from_allowlist(pool: &PgPool, community: CommunityId) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Bootstrap).await?; // Check if pubkey_allowlist table exists. let exists: bool = sqlx::query_scalar( "SELECT EXISTS (SELECT 1 FROM information_schema.tables \ WHERE table_schema = 'public' AND table_name = 'pubkey_allowlist')", ) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; if !exists { @@ -591,7 +642,7 @@ pub async fn backfill_from_allowlist(pool: &PgPool, community: CommunityId) -> R let has_members: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM relay_members WHERE community_id = $1)") .bind(community.as_uuid()) - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; if has_members { @@ -606,7 +657,7 @@ pub async fn backfill_from_allowlist(pool: &PgPool, community: CommunityId) -> R ON CONFLICT (community_id, pubkey) DO NOTHING", ) .bind(community.as_uuid()) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected()) @@ -624,7 +675,14 @@ impl Db { #[datastore_span(name = "is_relay_member", system = "postgresql")] pub async fn is_relay_member(&self, community: CommunityId, pubkey: &str) -> Result { let path = "relay_membership"; - match self.route_read(path, RoutePredicate::Bounded).await { + match self + .route_read( + path, + RoutePredicate::Bounded, + crate::observability::ReaderOperation::Authorization, + ) + .await + { RouteDecision::Replica(mut tx, _entry, reason) => { match is_relay_member_on(&mut tx, community, pubkey).await { Ok(is_member) => { @@ -738,6 +796,18 @@ impl Db { bootstrap_owner(&self.pool, community, owner_pubkey).await } + /// Ensure an owner during operator-driven community provisioning. + #[datastore_span(name = "provision_owner", system = "postgresql")] + pub async fn provision_owner(&self, community: CommunityId, owner_pubkey: &str) -> Result<()> { + bootstrap_owner_with_operation( + &self.pool, + community, + owner_pubkey, + observability::WriterOperation::Authorization, + ) + .await + } + /// Returns `true` if any member of `community` holds the `admin` or /// `owner` role. #[datastore_span(name = "has_admin_or_owner", system = "postgresql")] @@ -784,23 +854,80 @@ impl Db { name = "nip43_membership_snapshot_needs_reconciliation", system = "postgresql" )] + #[deprecated( + note = "use nip43_membership_snapshot_needs_reconciliation_for_bootstrap or nip43_membership_snapshot_needs_reconciliation_for_maintenance" + )] pub async fn nip43_membership_snapshot_needs_reconciliation( &self, community_id: CommunityId, relay_pubkey: &nostr::PublicKey, ) -> Result { - let snapshot = self - .query_events(&crate::event::EventQuery { + self.nip43_membership_snapshot_needs_reconciliation_with_operation( + community_id, + relay_pubkey, + observability::WriterOperation::Maintenance, + ) + .await + } + + /// Startup-attributed variant of the NIP-43 snapshot comparison. + #[datastore_span( + name = "nip43_membership_snapshot_needs_reconciliation_for_bootstrap", + system = "postgresql" + )] + pub async fn nip43_membership_snapshot_needs_reconciliation_for_bootstrap( + &self, + community_id: CommunityId, + relay_pubkey: &nostr::PublicKey, + ) -> Result { + self.nip43_membership_snapshot_needs_reconciliation_with_operation( + community_id, + relay_pubkey, + observability::WriterOperation::Bootstrap, + ) + .await + } + + /// Periodic maintenance variant of the NIP-43 snapshot comparison. + #[datastore_span( + name = "nip43_membership_snapshot_needs_reconciliation_for_maintenance", + system = "postgresql" + )] + pub async fn nip43_membership_snapshot_needs_reconciliation_for_maintenance( + &self, + community_id: CommunityId, + relay_pubkey: &nostr::PublicKey, + ) -> Result { + self.nip43_membership_snapshot_needs_reconciliation_with_operation( + community_id, + relay_pubkey, + observability::WriterOperation::Maintenance, + ) + .await + } + + async fn nip43_membership_snapshot_needs_reconciliation_with_operation( + &self, + community_id: CommunityId, + relay_pubkey: &nostr::PublicKey, + operation: observability::WriterOperation, + ) -> Result { + let snapshot = crate::event::query_events_with_operation( + &self.pool, + &crate::event::EventQuery { kinds: Some(vec![buzz_core::kind::KIND_NIP43_MEMBERSHIP_LIST as i32]), pubkey: Some(relay_pubkey.to_bytes().to_vec()), global_only: true, limit: Some(1), ..crate::event::EventQuery::for_community(community_id) - }) - .await? - .into_iter() - .next(); - let members = self.list_relay_members(community_id).await?; + }, + operation, + ) + .await? + .into_iter() + .next(); + let members = + list_relay_members_with_operation(&self.pool, community_id, operation).await?; let Some(snapshot) = snapshot else { return Ok(true); diff --git a/crates/buzz-db/src/store/relay_operators.rs b/crates/buzz-db/src/store/relay_operators.rs index 204e88fd95a..41b45a9ac68 100644 --- a/crates/buzz-db/src/store/relay_operators.rs +++ b/crates/buzz-db/src/store/relay_operators.rs @@ -97,7 +97,12 @@ pub async fn upsert( added_by: &[u8], config_operator_exists: bool, ) -> Result<()> { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // A demotion to moderator can drop the effective-operator count; serialize // it against every other operator-removing mutation via the roster-wide @@ -193,7 +198,12 @@ pub async fn remove( actor: &[u8], config_operator_exists: bool, ) -> Result { - let mut tx = pool.begin().await?; + let connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; + let mut tx = sqlx::Transaction::begin(connection, None).await?; // Serialize against every other operator-removing mutation before the // delete so the post-delete count reflects a stable roster. @@ -235,11 +245,16 @@ pub async fn remove( /// Fetch one relay operator/moderator row by pubkey. pub async fn get(pool: &PgPool, pubkey: &[u8]) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( "SELECT pubkey, role, added_by, created_at FROM relay_operators WHERE pubkey = $1", ) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map( @@ -258,10 +273,15 @@ pub async fn get(pool: &PgPool, pubkey: &[u8]) -> Result Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let rows = sqlx::query( "SELECT pubkey, role, added_by, created_at FROM relay_operators ORDER BY created_at ASC", ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; rows.into_iter() diff --git a/crates/buzz-db/src/store/replaceable.rs b/crates/buzz-db/src/store/replaceable.rs index 19f0d2d008e..5985d22b6d0 100644 --- a/crates/buzz-db/src/store/replaceable.rs +++ b/crates/buzz-db/src/store/replaceable.rs @@ -1039,7 +1039,7 @@ mod postgres_tests { ); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin caller transaction"); let result = db @@ -1109,7 +1109,10 @@ mod postgres_tests { .1 ); - let mut tx = db.begin_transaction().await.expect("begin replacement tx"); + let mut tx = db + .begin_event_write_transaction() + .await + .expect("begin replacement tx"); let outcome = db .replace_parameterized_event_in_transaction( &mut tx, @@ -1143,7 +1146,7 @@ mod postgres_tests { assert_eq!(live_id, old.id.as_bytes().to_vec()); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin stale revision tx"); let mismatch = db @@ -1177,7 +1180,7 @@ mod postgres_tests { .sign_with_keys(&keys) .expect("sign missing project"); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin missing revision tx"); let missing_result = db @@ -1255,7 +1258,7 @@ mod postgres_tests { .expect("install failure injection"); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin caller transaction"); let error = db @@ -1336,7 +1339,7 @@ mod postgres_tests { .expect("soft-delete duplicate row"); let mut seed_tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin seed transaction"); let (_, was_inserted) = @@ -1347,7 +1350,7 @@ mod postgres_tests { seed_tx.commit().await.expect("commit older live head"); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .expect("begin caller transaction"); let result = db diff --git a/crates/buzz-db/src/store/thread.rs b/crates/buzz-db/src/store/thread.rs index 0cf4e91f342..a38ac2b0380 100644 --- a/crates/buzz-db/src/store/thread.rs +++ b/crates/buzz-db/src/store/thread.rs @@ -11,6 +11,23 @@ use uuid::Uuid; use buzz_datastore_tracing::datastore_span; +async fn acquire_event_write_connection( + pool: &PgPool, +) -> Result> { + Ok(crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?) +} + +async fn begin_event_write_transaction( + pool: &PgPool, +) -> Result> { + let connection = acquire_event_write_connection(pool).await?; + Ok(sqlx::Transaction::begin(connection, None).await?) +} + use buzz_core::CommunityId; use crate::{ @@ -131,7 +148,7 @@ pub async fn insert_thread_metadata( depth: i32, broadcast: bool, ) -> Result<()> { - let mut tx = pool.begin().await?; + let mut tx = begin_event_write_transaction(pool).await?; let result = sqlx::query( r#" @@ -259,6 +276,7 @@ pub async fn increment_reply_count( parent_event_id: &[u8], root_event_id: Option<&[u8]>, ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; // Always bump the parent's direct reply count and last-reply timestamp. sqlx::query( r#" @@ -270,7 +288,7 @@ pub async fn increment_reply_count( ) .bind(community_id.as_uuid()) .bind(parent_event_id) - .execute(pool) + .execute(&mut *connection) .await?; // Always bump root's descendant_count, regardless of whether root == parent. @@ -284,7 +302,7 @@ pub async fn increment_reply_count( ) .bind(community_id.as_uuid()) .bind(root_id) - .execute(pool) + .execute(&mut *connection) .await?; } @@ -300,6 +318,7 @@ pub async fn decrement_reply_count( parent_event_id: &[u8], root_event_id: Option<&[u8]>, ) -> Result<()> { + let mut connection = acquire_event_write_connection(pool).await?; // Always decrement the parent's direct reply count (floor at 0). sqlx::query( r#" @@ -310,7 +329,7 @@ pub async fn decrement_reply_count( ) .bind(community_id.as_uuid()) .bind(parent_event_id) - .execute(pool) + .execute(&mut *connection) .await?; // Always decrement root's descendant_count, regardless of whether root == parent. @@ -324,7 +343,7 @@ pub async fn decrement_reply_count( ) .bind(community_id.as_uuid()) .bind(root_id) - .execute(pool) + .execute(&mut *connection) .await?; } @@ -355,7 +374,11 @@ pub async fn get_thread_replies( limit: u32, cursor: Option<&[u8]>, ) -> Result> { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; get_thread_replies_on( &mut conn, community_id, @@ -520,6 +543,11 @@ pub async fn get_thread_summary( community_id: CommunityId, event_id: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let row = sqlx::query( r#" SELECT reply_count, descendant_count, last_reply_at @@ -530,7 +558,7 @@ pub async fn get_thread_summary( ) .bind(community_id.as_uuid()) .bind(event_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; let row = match row { @@ -563,7 +591,7 @@ pub async fn get_thread_summary( ) .bind(community_id.as_uuid()) .bind(event_id) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; let participants: Vec> = participant_rows @@ -599,7 +627,11 @@ pub async fn get_channel_window( cursor: Option<(DateTime, Vec)>, kind_filter: Option<&[u32]>, ) -> Result { - let mut conn = pool.acquire().await?; + let mut conn = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::SubscriptionHistory, + ) + .await?; get_channel_window_on( &mut conn, community_id, @@ -811,6 +843,11 @@ pub async fn get_thread_metadata_by_event( community_id: CommunityId, event_id: &[u8], ) -> Result> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::EventWrite, + ) + .await?; let row = sqlx::query( r#" SELECT @@ -830,7 +867,7 @@ pub async fn get_thread_metadata_by_event( ) .bind(community_id.as_uuid()) .bind(event_id) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; let row = match row { @@ -935,8 +972,13 @@ impl Db { ), None => ("thread_head", RoutePredicate::Bounded), }; - if let RouteDecision::Replica(mut tx, entry, reason) = - self.route_read(path, predicate).await + if let RouteDecision::Replica(mut tx, entry, reason) = self + .route_read( + path, + predicate, + crate::observability::ReaderOperation::SubscriptionHistory, + ) + .await { match crate::thread::get_thread_replies_on( &mut tx, @@ -1062,6 +1104,7 @@ impl Db { .route_read( path, RoutePredicate::from_channel_cursor(channel_id, &cursor), + crate::observability::ReaderOperation::SubscriptionHistory, ) .await { diff --git a/crates/buzz-db/src/store/usage.rs b/crates/buzz-db/src/store/usage.rs index ce581561bd5..9d557c8b18a 100644 --- a/crates/buzz-db/src/store/usage.rs +++ b/crates/buzz-db/src/store/usage.rs @@ -43,8 +43,10 @@ impl UsageMetricsLeader { /// Total number of communities registered on this relay. pub async fn community_count(pool: &PgPool) -> Result { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let row = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM communities") - .fetch_one(pool) + .fetch_one(&mut *connection) .await?; Ok(row) } @@ -64,6 +66,8 @@ pub struct CommunityUserCounts { /// /// Agent discriminator: `agent_owner_pubkey IS NOT NULL`. pub async fn user_counts(pool: &PgPool) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; // Single GROUP BY query; two conditional SUMs avoid two round-trips. let rows = sqlx::query_as::<_, (Uuid, i64, i64)>( r#" @@ -76,7 +80,7 @@ pub async fn user_counts(pool: &PgPool) -> Result> { GROUP BY community_id "#, ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -102,6 +106,8 @@ pub struct CommunityChannelCount { /// Return non-deleted channel counts per community per type. pub async fn channel_counts(pool: &PgPool) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, String, i64)>( r#" SELECT community_id, channel_type::text, COUNT(*) AS count @@ -110,7 +116,7 @@ pub async fn channel_counts(pool: &PgPool) -> Result> GROUP BY community_id, channel_type "#, ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -136,6 +142,8 @@ pub struct CommunityMessageCount { /// Return non-deleted kind=9 event counts per community. pub async fn message_counts(pool: &PgPool) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, i64)>( r#" SELECT community_id, COUNT(*) AS count @@ -144,7 +152,7 @@ pub async fn message_counts(pool: &PgPool) -> Result> GROUP BY community_id "#, ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -169,6 +177,8 @@ pub struct CommunityMemberCount { /// Return relay-member counts per community per role. pub async fn relay_member_counts(pool: &PgPool) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, String, i64)>( r#" SELECT community_id, role::text, COUNT(*) AS count @@ -176,7 +186,7 @@ pub async fn relay_member_counts(pool: &PgPool) -> Result Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, String, i64)>( r#" SELECT community_id, status::text, COUNT(*) AS count @@ -209,7 +221,7 @@ pub async fn workflow_counts(pool: &PgPool) -> Result Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let rows = sqlx::query_as::<_, (Uuid, i64)>( r#" SELECT community_id, COUNT(*) AS count @@ -240,7 +254,7 @@ pub async fn git_repo_counts(pool: &PgPool) -> Result GROUP BY community_id "#, ) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -280,6 +294,8 @@ pub async fn active_user_counts( pool: &PgPool, interval_sql: &'static str, ) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; // LEFT JOIN users: pubkeys with no row have u.* = NULL. // Three-way classification: // human — row exists (u.pubkey IS NOT NULL) and agent_owner_pubkey IS NULL @@ -304,7 +320,7 @@ pub async fn active_user_counts( "# ); let rows = sqlx::query_as::<_, (Uuid, i64, i64, i64)>(sqlx::AssertSqlSafe(sql)) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -334,6 +350,8 @@ pub async fn active_channel_counts( pool: &PgPool, interval_sql: &'static str, ) -> Result> { + let mut connection = + observability::acquire_writer(pool, observability::WriterOperation::Maintenance).await?; let sql = format!( r#" SELECT community_id, COUNT(DISTINCT channel_id) AS count @@ -346,7 +364,7 @@ pub async fn active_channel_counts( "# ); let rows = sqlx::query_as::<_, (Uuid, i64)>(sqlx::AssertSqlSafe(sql)) - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows @@ -370,8 +388,16 @@ pub struct CommunityHost { /// Fetch all community id → host mappings in one query. pub async fn community_hosts(pool: &PgPool) -> Result> { + community_hosts_with_operation(pool, observability::WriterOperation::Maintenance).await +} + +async fn community_hosts_with_operation( + pool: &PgPool, + operation: observability::WriterOperation, +) -> Result> { + let mut connection = observability::acquire_writer(pool, operation).await?; let rows = sqlx::query_as::<_, (Uuid, String)>("SELECT id, host FROM communities") - .fetch_all(pool) + .fetch_all(&mut *connection) .await?; Ok(rows .into_iter() @@ -391,8 +417,11 @@ impl Db { &self, lock_key: i64, ) -> Result> { - let mut connection = - observability::acquire(&self.pool, observability::PoolRole::Writer).await?; + let mut connection = observability::acquire_writer_with_legacy_metrics( + &self.pool, + observability::WriterOperation::Maintenance, + ) + .await?; let acquired = sqlx::query_scalar::<_, bool>("SELECT pg_try_advisory_lock($1)") .bind(lock_key) .fetch_one(&mut *connection) @@ -473,6 +502,12 @@ impl Db { pub async fn usage_community_hosts(&self) -> Result> { community_hosts(&self.pool).await } + + /// Return community host mappings during startup bootstrap work. + #[datastore_span(name = "bootstrap_community_hosts", system = "postgresql")] + pub async fn bootstrap_community_hosts(&self) -> Result> { + community_hosts_with_operation(&self.pool, observability::WriterOperation::Bootstrap).await + } } #[cfg(test)] diff --git a/crates/buzz-db/src/store/user.rs b/crates/buzz-db/src/store/user.rs index 67f9cb341dc..759e916e4b4 100644 --- a/crates/buzz-db/src/store/user.rs +++ b/crates/buzz-db/src/store/user.rs @@ -42,6 +42,22 @@ pub struct UserSearchProfile { /// The `true` case is the reliable signal for "user was just registered" — used /// by callers to increment `buzz_users_created_total`. pub async fn ensure_user(pool: &PgPool, community_id: CommunityId, pubkey: &[u8]) -> Result { + ensure_user_with_operation( + pool, + community_id, + pubkey, + crate::observability::WriterOperation::EventWrite, + ) + .await +} + +async fn ensure_user_with_operation( + pool: &PgPool, + community_id: CommunityId, + pubkey: &[u8], + operation: crate::observability::WriterOperation, +) -> Result { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; let result = sqlx::query( r#" INSERT INTO users (community_id, pubkey) @@ -51,7 +67,7 @@ pub async fn ensure_user(pool: &PgPool, community_id: CommunityId, pubkey: &[u8] ) .bind(community_id.as_uuid()) .bind(pubkey) - .execute(pool) + .execute(&mut *connection) .await?; Ok(result.rows_affected() == 1) } @@ -296,6 +312,24 @@ pub async fn set_agent_owner( agent_pubkey: &[u8], owner_pubkey: &[u8], ) -> Result { + set_agent_owner_with_operation( + pool, + community_id, + agent_pubkey, + owner_pubkey, + crate::observability::WriterOperation::EventWrite, + ) + .await +} + +async fn set_agent_owner_with_operation( + pool: &PgPool, + community_id: CommunityId, + agent_pubkey: &[u8], + owner_pubkey: &[u8], + operation: crate::observability::WriterOperation, +) -> Result { + let mut connection = crate::observability::acquire_writer(pool, operation).await?; // Conditional UPDATE: only set owner if currently NULL. This makes // "first mint wins" atomic — no TOCTOU race between concurrent mints. let result = sqlx::query( @@ -304,7 +338,7 @@ pub async fn set_agent_owner( .bind(owner_pubkey) .bind(community_id.as_uuid()) .bind(agent_pubkey) - .execute(pool) + .execute(&mut *connection) .await?; if result.rows_affected() == 0 { @@ -313,7 +347,7 @@ pub async fn set_agent_owner( let exists = sqlx::query(r#"SELECT 1 FROM users WHERE community_id = $1 AND pubkey = $2"#) .bind(community_id.as_uuid()) .bind(agent_pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; if exists.is_none() { return Err(crate::error::DbError::NotFound( @@ -334,12 +368,17 @@ pub async fn get_agent_channel_policy( community_id: CommunityId, pubkey: &[u8], ) -> Result>)>> { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query( r#"SELECT channel_add_policy::text AS channel_add_policy, agent_owner_pubkey FROM users WHERE community_id = $1 AND pubkey = $2"#, ) .bind(community_id.as_uuid()) .bind(pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; row.map(|r| -> Result<(String, Option>)> { @@ -359,13 +398,18 @@ pub async fn is_agent_owner( target_pubkey: &[u8], actor_pubkey: &[u8], ) -> Result { + let mut connection = crate::observability::acquire_writer( + pool, + crate::observability::WriterOperation::Authorization, + ) + .await?; let row = sqlx::query_scalar::<_, bool>( "SELECT agent_owner_pubkey = $3 FROM users WHERE community_id = $1 AND pubkey = $2 AND agent_owner_pubkey IS NOT NULL", ) .bind(community_id.as_uuid()) .bind(target_pubkey) .bind(actor_pubkey) - .fetch_optional(pool) + .fetch_optional(&mut *connection) .await?; Ok(row.unwrap_or(false)) } @@ -411,6 +455,23 @@ impl Db { crate::user::ensure_user(&self.pool, community_id, pubkey).await } + /// Ensure a principal while materializing an authenticated NIP-OA + /// authorization relationship. + #[datastore_span(name = "ensure_user_for_authorization", system = "postgresql")] + pub async fn ensure_user_for_authorization( + &self, + community_id: CommunityId, + pubkey: &[u8], + ) -> Result { + ensure_user_with_operation( + &self.pool, + community_id, + pubkey, + crate::observability::WriterOperation::Authorization, + ) + .await + } + /// Get a single user record by pubkey. #[datastore_span(name = "get_user", system = "postgresql")] pub async fn get_user( @@ -478,6 +539,25 @@ impl Db { crate::user::set_agent_owner(&self.pool, community_id, agent_pubkey, owner_pubkey).await } + /// Materialize an authenticated NIP-OA agent-owner relationship under + /// authorization attribution. + #[datastore_span(name = "set_agent_owner_for_authorization", system = "postgresql")] + pub async fn set_agent_owner_for_authorization( + &self, + community_id: CommunityId, + agent_pubkey: &[u8], + owner_pubkey: &[u8], + ) -> Result { + set_agent_owner_with_operation( + &self.pool, + community_id, + agent_pubkey, + owner_pubkey, + crate::observability::WriterOperation::Authorization, + ) + .await + } + /// Get the channel_add_policy and agent_owner_pubkey for a user. #[datastore_span(name = "get_agent_channel_policy", system = "postgresql")] pub async fn get_agent_channel_policy( diff --git a/crates/buzz-db/tests/observability_source.rs b/crates/buzz-db/tests/observability_source.rs index 9e37186009f..832b05c56f2 100644 --- a/crates/buzz-db/tests/observability_source.rs +++ b/crates/buzz-db/tests/observability_source.rs @@ -79,3 +79,549 @@ fn relay_admin_db_wrappers_have_exactly_one_datastore_span() { ); } } + +#[test] +fn p0_pool_acquisitions_use_typed_operation_pairs_without_other() { + let observability = include_str!("../src/runtime/observability.rs"); + assert!(observability.contains("enum PoolOperation")); + assert!(observability.contains("pub(crate) enum WriterOperation")); + assert!(observability.contains("pub(crate) enum ReaderOperation")); + assert!(observability.contains("Self::WriterAuthentication")); + assert!(observability.contains("Self::ReaderSubscriptionHistory")); + assert!(observability.contains("pub(crate) async fn acquire_writer(")); + assert!(observability.contains("pub(super) async fn acquire_reader_with_legacy_metrics(")); + assert!(observability.contains("static POOL_WAITERS: [Mutex")); + assert!(!observability.contains("AtomicU64")); + assert!(!observability.contains("DbOperation::Other")); + assert!(!observability.contains("\"other\"")); + assert!(!observability.contains("buzz_db_pool_acquire_timeouts_total")); + assert!(!observability.contains("\"result\" =>")); + let legacy_transaction = observability + .split_once("pub(crate) async fn begin_transaction(") + .expect("observability must expose attributed transaction acquisition") + .1 + .split_once("pub(crate) async fn observe_advisory_lock") + .expect("transaction acquisition must precede advisory-lock observation") + .0; + assert!(legacy_transaction.contains("acquire_writer_with_legacy_metrics(")); + + let runtime = include_str!("../src/runtime/mod.rs"); + assert!(runtime.contains("observability::acquire_writer_until(")); + assert!(runtime.contains("WriterOperation::Readiness")); + assert!(runtime.contains("WriterOperation::EventWrite")); + assert!(runtime.contains("ReaderOperation::Bootstrap")); + assert!(runtime.contains("pub async fn begin_event_write_transaction")); + let reader_boot = runtime + .split_once("async fn read_pool_boot_ping_once(") + .expect("runtime must expose the reader boot probe") + .1 + .split_once("#[cfg(test)]") + .expect("reader boot probe must precede its test seam") + .0; + assert!(reader_boot.contains("acquire_reader_with_legacy_metrics(")); + let routed_reader = runtime + .split_once("async fn proved_reader(") + .expect("runtime must expose the routed-reader checkout") + .1 + .split_once("async fn reader_aurora_capability_on(") + .expect("routed-reader checkout must precede capability probing") + .0; + assert!(routed_reader.contains("acquire_reader_with_legacy_metrics(read_pool, operation)")); + let event_write_transaction = runtime + .split_once("pub async fn begin_event_write_transaction(") + .expect("runtime must expose the legacy event-write transaction seam") + .1 + .split_once("pub async fn insert_event_with_serving_write_guard(") + .expect("legacy event-write transaction must precede guarded writes") + .0; + assert!(event_write_transaction.contains("acquire_writer_with_legacy_metrics(")); + + let migration = include_str!("../src/runtime/migration.rs"); + let migration_lock = migration + .split_once("pub(crate) async fn with_exclusive_schema_destruction_lock") + .expect("migration must expose the schema-safety acquisition seam") + .1 + .split_once("async fn reject_legacy_nip_rs_cardinality_ambiguity") + .expect("schema-safety acquisition must precede migration validation") + .0; + assert!(migration_lock.contains("acquire_writer_with_legacy_metrics(")); + + let allowlist = include_str!("../src/store/allowlist.rs"); + assert!(allowlist.contains("WriterOperation::Authentication")); + assert!(allowlist.contains("WriterOperation::Authorization")); + assert!(!allowlist.contains("fetch_one(&self.pool)")); + + let event = include_str!("../src/store/event.rs"); + assert!(event.contains("query_events_with_operation")); + assert!(event.contains("WriterOperation::Authorization")); + assert!(event.contains("WriterOperation::SubscriptionHistory")); + assert!(event.contains("ReaderOperation::SubscriptionHistory")); + let backfill_d_tags = event + .split_once("pub async fn backfill_d_tags") + .expect("event store must expose the startup d-tag backfill") + .1 + .split_once("/// Soft-delete NIP-29 discovery events") + .expect("d-tag backfill must precede discovery deletion") + .0; + assert!(backfill_d_tags.contains("WriterOperation::Bootstrap")); + assert!(backfill_d_tags.contains("execute(&mut *connection)")); + let soft_delete_discovery = event + .split_once("pub async fn soft_delete_discovery_events") + .expect("event store must expose discovery-event deletion") + .1 + .split_once("\n}\n\n#[cfg(test)]") + .expect("discovery deletion must end the production Db implementation") + .0; + assert!(soft_delete_discovery.contains("WriterOperation::EventWrite")); + assert!(soft_delete_discovery.contains("execute(&mut *connection)")); + + let side_effects = include_str!("../../buzz-relay/src/handlers/side_effects.rs"); + assert!(side_effects.contains("query_events_for_event_write")); + assert!(side_effects.contains("query_events_for_bootstrap")); + assert!(side_effects.contains(".list_channels_for_bootstrap(")); + + let deletion = include_str!("../src/store/deletion.rs"); + let public_serving_catalog = deletion + .split_once("pub async fn validate_serving_catalog(&self)") + .expect("deletion store must preserve its public serving-catalog API") + .1 + .split_once("async fn validate_serving_catalog_on") + .expect("public serving-catalog validation must delegate to its connection helper") + .0; + assert!(public_serving_catalog.contains("WriterOperation::Bootstrap")); + assert!(public_serving_catalog.contains("observability::acquire_writer(")); + assert!(public_serving_catalog.contains("validate_serving_catalog_on")); + assert!(!public_serving_catalog.contains("self.pool.acquire()")); + + let thread = include_str!("../src/store/thread.rs"); + let thread_metadata = thread + .split_once("pub async fn get_thread_metadata_by_event(") + .expect("thread store must expose metadata lookup") + .1 + .split_once("// -- Db API") + .expect("metadata lookup must precede the Db wrapper section") + .0; + assert!(thread_metadata.contains("WriterOperation::EventWrite")); + assert!(thread_metadata.contains("fetch_optional(&mut *connection)")); + assert!(!thread_metadata.contains("fetch_optional(pool)")); + + let channel = include_str!("../src/store/channel.rs"); + assert!(channel.contains("async fn begin_event_write_transaction(")); + assert!(channel.contains("async fn acquire_event_write_connection(")); + for (start, end, expected) in [ + ( + "pub async fn create_channel(\n", + "/// Creates a channel with a client-supplied UUID", + "begin_event_write_transaction(pool)", + ), + ( + "pub async fn create_channel_with_id(\n", + "/// Fetches a channel record by `(community_id, id)`", + "begin_event_write_transaction(pool)", + ), + ( + "pub async fn update_channel(\n", + "/// Sets the topic for a channel", + "begin_event_write_transaction(pool)", + ), + ( + "pub async fn set_topic(\n", + "/// Sets the purpose for a channel", + "acquire_event_write_connection(pool)", + ), + ( + "pub async fn set_purpose(\n", + "/// Archives a channel", + "acquire_event_write_connection(pool)", + ), + ( + "pub async fn archive_channel(\n", + "/// Unarchives a channel", + "acquire_event_write_connection(pool)", + ), + ( + "pub async fn unarchive_channel(\n", + "/// Soft-delete a channel", + "acquire_event_write_connection(pool)", + ), + ( + "pub async fn soft_delete_channel(\n", + "/// Archive ephemeral channels", + "acquire_event_write_connection(pool)", + ), + ] { + let function = channel + .split_once(start) + .unwrap_or_else(|| panic!("missing channel seam {start}")) + .1 + .split_once(end) + .unwrap_or_else(|| panic!("channel seam {start} must precede {end}")) + .0; + assert!( + function.contains(expected), + "channel seam {start} must use {expected}" + ); + assert!(!function.contains("pool.begin().await")); + assert!(!function.contains(".execute(pool)")); + assert!(!function.contains(".fetch_optional(pool)")); + } + let get_channel = channel + .split_once("async fn get_channel_with_operation(") + .expect("channel store must route shared lookups through caller-owned intent") + .1 + .split_once("/// Returns the canvas content") + .expect("channel lookup helper must precede canvas reads") + .0; + assert!(get_channel.contains("acquire_writer(pool, operation)")); + assert!(get_channel.contains("fetch_optional(&mut *connection)")); + assert!(!get_channel.contains("fetch_optional(pool)")); + assert!(channel.contains("pub async fn get_channel_for_event_write(")); + let list_channels = channel + .split_once("async fn list_channels_with_operation(") + .expect("channel listing must accept caller-owned intent") + .1 + .split_once("/// A channel archived by the ephemeral-channel reaper") + .expect("channel listing must precede ephemeral-channel types") + .0; + assert!(list_channels.contains("acquire_writer(pool, operation)")); + assert!(list_channels.contains("fetch_all(&mut *connection)")); + assert!(!list_channels.contains("fetch_all(pool)")); + assert!(channel.contains("pub async fn list_channels_for_bootstrap(")); + + let channel_members = include_str!("../src/store/channel_members.rs"); + assert!(channel_members.contains("async fn get_members_with_operation(")); + assert!(channel_members.contains("pub async fn get_members_for_event_write(")); + assert!(channel_members.contains("async fn get_users_bulk_with_operation(")); + assert!(channel_members.contains("pub async fn get_users_bulk_for_event_write(")); + + let huddle_link = event + .split_once("async fn huddle_started_link_exists_with_operation(") + .expect("huddle link lookup must accept caller-owned intent") + .1 + .split_once("/// Insert a Nostr event") + .expect("huddle link lookup must precede event insertion") + .0; + assert!(huddle_link.contains("acquire_writer(pool, operation)")); + assert!(event.contains("pub async fn huddle_started_link_exists_for_event_write(")); + let ingest = include_str!("../../buzz-relay/src/handlers/ingest.rs"); + assert!(ingest.contains(".huddle_started_link_exists_for_event_write(")); + let audio = include_str!("../../buzz-relay/src/audio/handler.rs"); + assert!(audio.contains(".huddle_started_link_exists(")); + + let workflow_sink = include_str!("../../buzz-relay/src/workflow_sink.rs"); + assert!(workflow_sink.contains(".get_members_for_event_write(")); + assert!(workflow_sink.contains(".get_users_bulk_for_event_write(")); + + for write_caller in [ + include_str!("../../buzz-relay/src/handlers/side_effects.rs"), + include_str!("../../buzz-relay/src/handlers/ingest.rs"), + include_str!("../../buzz-relay/src/handlers/command_executor.rs"), + workflow_sink, + ] { + assert!(!write_caller.contains(".get_channel(")); + assert!(write_caller.contains(".get_channel_for_event_write(")); + } + + let user = include_str!("../src/store/user.rs"); + let agent_channel_policy = user + .split_once("pub async fn get_agent_channel_policy(") + .expect("user store must expose get_agent_channel_policy") + .1 + .split_once("/// Check whether `actor_pubkey`") + .expect("agent policy lookup must precede owner lookup") + .0; + assert!(agent_channel_policy.contains("WriterOperation::Authorization")); + assert!(agent_channel_policy.contains("fetch_optional(&mut *connection)")); + assert!(!agent_channel_policy.contains("fetch_optional(pool)")); + let is_agent_owner = user + .split_once("pub async fn is_agent_owner(") + .expect("user store must expose is_agent_owner") + .1 + .split_once("/// Set the channel_add_policy") + .expect("is_agent_owner must precede set_agent_channel_policy") + .0; + assert!(is_agent_owner.contains("WriterOperation::Authorization")); + assert!(is_agent_owner.contains("acquire_writer(")); + assert!(is_agent_owner.contains("fetch_optional(&mut *connection)")); + assert!(!is_agent_owner.contains("fetch_optional(pool)")); + + let moderation = include_str!("../src/store/moderation.rs"); + let restriction_state = moderation + .split_once("pub async fn restriction_state(") + .expect("moderation store must expose restriction_state") + .1 + .split_once("/// Fetch the full ban/timeout row") + .expect("restriction state must precede full ban reads") + .0; + assert!(restriction_state.contains("WriterOperation::Authorization")); + assert!(restriction_state.contains("fetch_optional(&mut *connection)")); + assert!(!restriction_state.contains("fetch_optional(pool)")); + + let community_store = include_str!("../src/store/community.rs"); + let ensure_community = community_store + .split_once("pub async fn ensure_configured_community(") + .expect("community store must expose ensure_configured_community") + .1 + .split_once("/// Atomically creates a community") + .expect("configured-community helpers must precede community creation") + .0; + assert!(ensure_community.contains("WriterOperation::Authorization")); + assert!(ensure_community.contains("WriterOperation::Bootstrap")); + assert!(ensure_community.contains("ensure_configured_community_with_operation")); + assert!(ensure_community.contains("acquire_writer(&self.pool, operation)")); + assert!(ensure_community.contains("fetch_optional(&mut *connection)")); + let management_lookup = community_store + .split_once("pub async fn lookup_community_by_host_for_management(") + .expect("community store must expose management host lookup") + .1 + .split_once("/// Lists communities where") + .expect("management lookup must precede owner listing") + .0; + assert!(management_lookup.contains("WriterOperation::Authorization")); + assert!(management_lookup.contains("fetch_optional(&mut *connection)")); + assert!(!management_lookup.contains("fetch_optional(&self.pool)")); + let community_production = community_store + .split("\n#[cfg(test)]") + .next() + .expect("community production source"); + for required in [ + "WriterOperation::TenantResolution", + "WriterOperation::Authorization", + "WriterOperation::SubscriptionHistory", + "WriterOperation::EventWrite", + ] { + assert!( + community_production.contains(required), + "community P0 paths must include {required} attribution" + ); + } + assert!(!community_production.contains("self.pool.begin().await")); + assert!(!community_production.contains(".fetch_one(&self.pool)")); + assert!(!community_production.contains(".fetch_all(&self.pool)")); + assert!(!community_production.contains(".execute(&self.pool)")); + assert_eq!( + community_production + .matches(".fetch_optional(&self.pool)") + .count(), + 1, + "only the out-of-scope NIP-11 metadata read may retain a raw pool checkout" + ); + + let thread_summary = thread + .split_once("pub async fn get_thread_summary(") + .expect("thread store must expose get_thread_summary") + .1 + .split_once("/// Fetch one channel window") + .expect("thread summary must precede channel-window reads") + .0; + assert!(thread_summary.contains("WriterOperation::EventWrite")); + assert!(thread_summary.contains("fetch_optional(&mut *connection)")); + assert!(thread_summary.contains("fetch_all(&mut *connection)")); + assert!(!thread_summary.contains("fetch_optional(pool)")); + assert!(!thread_summary.contains("fetch_all(pool)")); + + let archived_identities = include_str!("../src/store/archived_identities.rs"); + let archived_identity_production = archived_identities + .split("\n#[cfg(test)]") + .next() + .expect("archived identity production source"); + assert_eq!( + archived_identity_production + .matches("WriterOperation::EventWrite") + .count(), + 4, + "all four archived identity operations must be attributed to event writes" + ); + assert!(!archived_identity_production.contains("fetch_optional(pool)")); + assert!(!archived_identity_production.contains("fetch_all(pool)")); + assert!(!archived_identity_production.contains("execute(pool)")); + + let relay_main = include_str!("../../buzz-relay/src/main.rs"); + assert!(relay_main.contains("pool_state.db.refresh_pool_waiter_metrics();")); + assert!(relay_main.contains(".ensure_configured_community_for_bootstrap(")); + + let runtime = include_str!("../src/runtime/mod.rs"); + assert!(runtime.contains("observability::refresh_pool_waiters(self.read_pool.is_some())")); + assert!(runtime.contains("self.verify_replica_fence_at_boot().await?")); + let fence_boot = runtime + .split_once("pub(crate) async fn verify_replica_fence_at_boot") + .expect("runtime must expose attributed boot fence verification") + .1 + .split_once("/// The pool for lag-tolerant reads") + .expect("boot fence verification must precede routed-read plumbing") + .0; + assert!(fence_boot.contains("WriterOperation::Bootstrap")); + + let replica_fence = include_str!("../src/runtime/replica_fence.rs"); + let replica_fence_production = replica_fence + .split("\n#[cfg(test)]") + .next() + .expect("replica-fence production source"); + assert!(replica_fence_production.contains("WriterOperation::Bootstrap")); + assert!(replica_fence_production.contains("WriterOperation::Maintenance")); + assert!(!replica_fence_production.contains("pool.begin().await")); + assert!(!replica_fence_production.contains("writer.acquire().await")); + assert!(!replica_fence_production.contains("fetch_optional(writer)")); + + let usage = include_str!("../src/store/usage.rs"); + let usage_production = usage + .split("\n#[cfg(test)]") + .next() + .expect("usage production source"); + let usage_leader_lock = usage_production + .split_once("pub async fn try_lock_usage_metrics(") + .expect("usage store must expose the legacy leader-lock acquisition") + .1 + .split_once("pub async fn usage_community_count(") + .expect("usage leader lock must precede counter reads") + .0; + assert!(usage_leader_lock.contains("acquire_writer_with_legacy_metrics(")); + assert!( + usage_production + .matches("WriterOperation::Maintenance") + .count() + >= 11, + "every periodic usage checkout must be maintenance-attributed" + ); + for bypass in [ + ".fetch_one(pool)", + ".fetch_all(pool)", + ".fetch_optional(pool)", + ".execute(pool)", + ] { + assert!( + !usage_production.contains(bypass), + "usage production path bypasses operation attribution with {bypass}" + ); + } + + let channel_reaper = channel + .split_once("pub async fn reap_expired_ephemeral_channels(pool:") + .expect("channel store must expose ephemeral reaper") + .1 + .split_once("\nimpl Db {") + .expect("ephemeral reaper must precede Db wrappers") + .0; + assert!(channel_reaper.contains("WriterOperation::Maintenance")); + assert!(channel_reaper.contains("fetch_all(&mut *connection)")); + + let deletion = include_str!("../src/store/deletion.rs"); + let lease_reaper = deletion + .split_once("pub async fn reap_expired_serving_write_leases") + .expect("deletion store must expose serving-lease reaper") + .1 + .split_once("/// Return serving-lease counts") + .expect("serving-lease reaper must precede stats") + .0; + assert!(lease_reaper.contains("WriterOperation::Maintenance")); + assert!(lease_reaper.contains("execute(&mut *connection)")); + let lease_stats = deletion + .split_once("pub async fn serving_lease_stats") + .expect("deletion store must expose serving-lease stats") + .1 + .split_once("/// Whether a community remains active") + .expect("serving-lease stats must precede serving-state reads") + .0; + assert!(lease_stats.contains("WriterOperation::Maintenance")); + assert!(lease_stats.contains("fetch_one(&mut *connection)")); + for (start, end) in [ + ( + "pub async fn acquire_serving_write_lease", + "/// Renew an already-admitted external side-effect lease", + ), + ( + "pub async fn renew_serving_write_lease", + "/// Release a serving side-effect lease", + ), + ( + "pub async fn release_serving_write_lease", + "/// Check that an external side-effect lease remains current", + ), + ( + "pub async fn verify_serving_write_lease", + "/// Delete expired serving leases", + ), + ( + "pub async fn is_serving_active", + "async fn advance_with_checkpoint", + ), + ] { + let function = deletion + .split_once(start) + .unwrap_or_else(|| panic!("missing serving-write seam {start}")) + .1 + .split_once(end) + .unwrap_or_else(|| panic!("serving-write seam {start} must precede {end}")) + .0; + assert!( + function.contains("WriterOperation::EventWrite"), + "serving-write seam {start} must be event-write attributed" + ); + assert!(!function.contains("self.pool.begin().await")); + assert!(!function.contains(".execute(&self.pool)")); + assert!(!function.contains(".fetch_one(&self.pool)")); + } + + let ensure_authorization = user + .split_once("pub async fn ensure_user_for_authorization(") + .expect("user store must expose NIP-OA authorization ensure") + .1 + .split_once("/// Get a single user record") + .expect("authorization ensure must precede generic user reads") + .0; + assert!(ensure_authorization.contains("WriterOperation::Authorization")); + let set_owner_authorization = user + .split_once("pub async fn set_agent_owner_for_authorization(") + .expect("user store must expose NIP-OA authorization owner write") + .1 + .split_once("/// Get the channel_add_policy") + .expect("authorization owner write must precede policy reads") + .0; + assert!(set_owner_authorization.contains("WriterOperation::Authorization")); + let relay_api = include_str!("../../buzz-relay/src/api/mod.rs"); + assert!(relay_api.contains(".ensure_user_for_authorization(")); + assert!(relay_api.contains(".set_agent_owner_for_authorization(")); + + for (domain, source) in [ + ( + "channel_members", + include_str!("../src/store/channel_members.rs"), + ), + ("archived_identities", archived_identities), + ("event", event), + ("git_repo", include_str!("../src/store/git_repo.rs")), + ("push", include_str!("../src/store/push.rs")), + ("replica_fence", replica_fence), + ("reaction", include_str!("../src/store/reaction.rs")), + ("relay_invite", include_str!("../src/store/relay_invite.rs")), + ( + "relay_members", + include_str!("../src/store/relay_members.rs"), + ), + ("thread", thread), + ( + "relay_operators", + include_str!("../src/store/relay_operators.rs"), + ), + ("usage", usage), + ] { + let production = source.split("\n#[cfg(test)]").next().unwrap_or(source); + for bypass in [ + "pool.begin().await", + "self.pool.begin().await", + ".fetch_one(pool)", + ".fetch_one(&self.pool)", + ".fetch_all(pool)", + ".fetch_all(&self.pool)", + ".fetch_optional(pool)", + ".fetch_optional(&self.pool)", + ".execute(pool)", + ".execute(&self.pool)", + ] { + assert!( + !production.contains(bypass), + "{domain} production path bypasses operation attribution with {bypass}" + ); + } + } +} diff --git a/crates/buzz-relay/src/api/mod.rs b/crates/buzz-relay/src/api/mod.rs index b7a0458f5a8..5745b8d4e59 100644 --- a/crates/buzz-relay/src/api/mod.rs +++ b/crates/buzz-relay/src/api/mod.rs @@ -223,7 +223,7 @@ pub mod relay_members { for (role, pubkey) in [("agent", agent), ("owner", owner)] { match state .db - .ensure_user(tenant.community(), pubkey.as_bytes()) + .ensure_user_for_authorization(tenant.community(), pubkey.as_bytes()) .await { Ok(true) => { @@ -243,7 +243,11 @@ pub mod relay_members { let materialized = match state .db - .set_agent_owner(tenant.community(), agent.as_bytes(), owner.as_bytes()) + .set_agent_owner_for_authorization( + tenant.community(), + agent.as_bytes(), + owner.as_bytes(), + ) .await { Ok(true) => true, diff --git a/crates/buzz-relay/src/handlers/command_executor.rs b/crates/buzz-relay/src/handlers/command_executor.rs index 41d7900b7d1..074f6b391d0 100644 --- a/crates/buzz-relay/src/handlers/command_executor.rs +++ b/crates/buzz-relay/src/handlers/command_executor.rs @@ -109,7 +109,7 @@ async fn persist_command_event( let channel_id = channel_id_override.or_else(|| extract_channel_id(event)); let mut tx = db - .begin_transaction() + .begin_event_write_transaction() .await .map_err(|e| IngestError::Internal(format!("error: begin transaction: {e}")))?; buzz_deletion::store(db) @@ -463,7 +463,7 @@ async fn handle_dm_add_member( // 3. Validate channel is type "dm" let existing_channel = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map_err(|_| IngestError::Rejected("invalid: DM not found".into()))?; if existing_channel.channel_type != "dm" { @@ -473,7 +473,7 @@ async fn handle_dm_add_member( // 4. Get existing members, merge with new let existing_members = state .db - .get_members(tenant.community(), channel_id) + .get_members_for_event_write(tenant.community(), channel_id) .await .map_err(|e| IngestError::Internal(format!("error: get members: {e}")))?; @@ -593,7 +593,7 @@ async fn handle_dm_hide( // 3. Validate channel is type "dm" let channel = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map_err(|_| IngestError::Rejected("invalid: DM not found".into()))?; if channel.channel_type != "dm" { @@ -763,7 +763,7 @@ async fn handle_workflow_def( let community_id = tenant.community(); state .db - .get_channel(community_id, channel_id) + .get_channel_for_event_write(community_id, channel_id) .await .map_err(|_| IngestError::Rejected("invalid: workflow channel not found".into()))?; @@ -1595,7 +1595,10 @@ mod postgres_tests { "legacy-malformed", ); - let mut tx = db.begin_transaction().await.expect("begin legacy seed"); + let mut tx = db + .begin_event_write_transaction() + .await + .expect("begin legacy seed"); let (_, was_inserted) = buzz_db::event::insert_event_in_transaction( &mut tx, tenant.community(), diff --git a/crates/buzz-relay/src/handlers/community_provisioning.rs b/crates/buzz-relay/src/handlers/community_provisioning.rs index 3185af8bea0..229b5f37161 100644 --- a/crates/buzz-relay/src/handlers/community_provisioning.rs +++ b/crates/buzz-relay/src/handlers/community_provisioning.rs @@ -327,7 +327,7 @@ pub async fn provision_community( if let Some(owner_hex) = &initial_owner { state .db - .bootstrap_owner(record.id, owner_hex) + .provision_owner(record.id, owner_hex) .await .map_err(|e| format!("community provisioned but owner bootstrap failed: {e}"))?; publish_membership_snapshot_if_required(state, record.id, &record.host).await; diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index b4a3e24f8f4..ee1d0312be9 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -91,7 +91,7 @@ async fn validate_huddle_lifecycle_event( let backing_channel_id = huddle_backing_channel_id(event)?; let backing = state .db - .get_channel(tenant.community(), backing_channel_id) + .get_channel_for_event_write(tenant.community(), backing_channel_id) .await .map_err(map_huddle_backing_channel_error)?; let signer = event.pubkey.to_bytes(); @@ -122,7 +122,7 @@ async fn validate_huddle_lifecycle_event( })?; let linked = state .db - .huddle_started_link_exists( + .huddle_started_link_exists_for_event_write( tenant.community(), parent_channel_id, backing_channel_id, @@ -597,7 +597,10 @@ pub(crate) async fn derive_reaction_channel( _ => return ReactionChannelResult::NoTarget, }; - match db.get_event_by_id(community_id, &id_bytes).await { + match db + .get_event_by_id_for_event_write(community_id, &id_bytes) + .await + { Ok(Some(target)) => match target.channel_id { Some(ch_id) => ReactionChannelResult::Channel(ch_id), None => ReactionChannelResult::NoChannel, @@ -759,7 +762,7 @@ pub(crate) async fn check_channel_membership( Some(ch) => ch.visibility == "open", None => state .db - .get_channel(tenant.community(), ch_id) + .get_channel_for_event_write(tenant.community(), ch_id) .await .map(|ch| ch.visibility == "open") .unwrap_or(false), @@ -827,7 +830,9 @@ pub(crate) async fn resolve_nip10_thread_meta( hex::decode(&parent_hex).map_err(|_| "invalid parent event ID hex".to_string())?; let (parent_event_result, parent_meta_result) = tokio::join!( - state.db.get_event_by_id(community_id, &parent_bytes), + state + .db + .get_event_by_id_for_event_write(community_id, &parent_bytes), state .db .get_thread_metadata_by_event(community_id, &parent_bytes), @@ -863,7 +868,7 @@ pub(crate) async fn resolve_nip10_thread_meta( } let root_ts = if let Ok(Some(root_ev)) = state .db - .get_event_by_id(community_id, &effective_root) + .get_event_by_id_for_event_write(community_id, &effective_root) .await { chrono::DateTime::from_timestamp(root_ev.event.created_at.as_secs() as i64, 0) @@ -947,13 +952,16 @@ async fn derive_ancestry_from_parent_tags( if parent_root.as_slice() == parent_bytes { (parent_root, parent_created, 1) } else { - let root_created = - if let Ok(Some(root_ev)) = state.db.get_event_by_id(community_id, &parent_root).await { - chrono::DateTime::from_timestamp(root_ev.event.created_at.as_secs() as i64, 0) - .unwrap_or(parent_created) - } else { - parent_created - }; + let root_created = if let Ok(Some(root_ev)) = state + .db + .get_event_by_id_for_event_write(community_id, &parent_root) + .await + { + chrono::DateTime::from_timestamp(root_ev.event.created_at.as_secs() as i64, 0) + .unwrap_or(parent_created) + } else { + parent_created + }; (parent_root, root_created, 2) } } @@ -1019,7 +1027,9 @@ pub(crate) async fn resolve_relay_reply_thread_meta( hex::decode(parent_hex).map_err(|_| "invalid parent event ID hex".to_string())?; let (parent_event_result, parent_meta_result) = tokio::join!( - state.db.get_event_by_id(community_id, &parent_bytes), + state + .db + .get_event_by_id_for_event_write(community_id, &parent_bytes), state .db .get_thread_metadata_by_event(community_id, &parent_bytes), @@ -1053,7 +1063,7 @@ pub(crate) async fn resolve_relay_reply_thread_meta( parent_created } else if let Ok(Some(root_ev)) = state .db - .get_event_by_id(community_id, &effective_root) + .get_event_by_id_for_event_write(community_id, &effective_root) .await { chrono::DateTime::from_timestamp(root_ev.event.created_at.as_secs() as i64, 0) @@ -1163,7 +1173,7 @@ async fn validate_edit_ownership( hex::decode(&target_hex).map_err(|_| "invalid target event ID".to_string())?; let target_event = state .db - .get_event_by_id(community_id, &target_bytes) + .get_event_by_id_for_event_write(community_id, &target_bytes) .await .map_err(|e| format!("db error: {e}"))? .ok_or_else(|| "edit target event not found".to_string())?; @@ -1193,7 +1203,7 @@ async fn validate_edit_ownership( if !is_member { let is_open = state .db - .get_channel(community_id, ch_id) + .get_channel_for_event_write(community_id, ch_id) .await .map(|ch| ch.visibility == "open") .unwrap_or(false); @@ -1244,7 +1254,7 @@ async fn validate_forum_vote_target( hex::decode(&target_hex).map_err(|_| "invalid target event ID".to_string())?; let target_event = state .db - .get_event_by_id(community_id, &target_bytes) + .get_event_by_id_for_event_write(community_id, &target_bytes) .await .map_err(|e| format!("db error: {e}"))? .ok_or_else(|| "vote target event not found".to_string())?; @@ -2435,7 +2445,7 @@ async fn ingest_event_inner( })?; match state .db - .get_event_by_id(tenant.community(), &target_bytes) + .get_event_by_id_for_event_write(tenant.community(), &target_bytes) .await { Ok(Some(target)) => target.channel_id, @@ -2483,7 +2493,11 @@ async fn ingest_event_inner( // it later in this request); each gate keeps its existing missing-row // behavior. let channel_row = match channel_id { - Some(ch_id) => state.db.get_channel(tenant.community(), ch_id).await.ok(), + Some(ch_id) => state + .db + .get_channel_for_event_write(tenant.community(), ch_id) + .await + .ok(), None => None, }; // E1 phase-2 (§4.8 phase-2 addendum): resolve the fan-out visibility once, diff --git a/crates/buzz-relay/src/handlers/side_effects.rs b/crates/buzz-relay/src/handlers/side_effects.rs index d3416d673c5..8183fe98d80 100644 --- a/crates/buzz-relay/src/handlers/side_effects.rs +++ b/crates/buzz-relay/src/handlers/side_effects.rs @@ -147,7 +147,10 @@ async fn evict_non_member_channel_subscriptions( state: &Arc, channel_id: Uuid, ) -> anyhow::Result<()> { - let members = state.db.get_members(tenant.community(), channel_id).await?; + let members = state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await?; let member_pubkeys: std::collections::HashSet> = members.into_iter().map(|m| m.pubkey).collect(); @@ -262,7 +265,7 @@ pub async fn validate_standard_deletion_event( for target_id in target_ids { let target_event = state .db - .get_event_by_id_including_deleted(tenant.community(), &target_id) + .get_event_by_id_including_deleted_for_event_write(tenant.community(), &target_id) .await? .ok_or_else(|| anyhow::anyhow!("target event not found"))?; @@ -327,7 +330,7 @@ pub async fn validate_admin_event( // (unarchive), which must be allowed through so the channel can be restored. let channel = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map_err(|_| anyhow::anyhow!("channel not found"))?; let is_unarchive_request = kind == 9002 @@ -655,7 +658,7 @@ pub async fn validate_admin_event( // BEFORE storage. Fail closed: missing target → reject. let target_event = state .db - .get_event_by_id(tenant.community(), &target_id) + .get_event_by_id_for_event_write(tenant.community(), &target_id) .await .map_err(|e| anyhow::anyhow!("db error looking up target: {e}"))? .ok_or_else(|| anyhow::anyhow!("target event not found"))?; @@ -687,7 +690,7 @@ pub async fn validate_admin_event( } let is_open = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map(|ch| ch.visibility == "open") .unwrap_or(false); @@ -1015,7 +1018,7 @@ async fn emit_addressable_discovery_event( let min_ts = { let existing = state .db - .query_events(&buzz_db::event::EventQuery { + .query_events_for_event_write(&buzz_db::event::EventQuery { kinds: Some(vec![kind as i32]), channel_id: Some(channel_id), limit: Some(1), @@ -1129,8 +1132,14 @@ pub async fn emit_group_discovery_events( state: &Arc, channel_id: Uuid, ) -> anyhow::Result<()> { - let channel = state.db.get_channel(tenant.community(), channel_id).await?; - let members = state.db.get_members(tenant.community(), channel_id).await?; + let channel = state + .db + .get_channel_for_event_write(tenant.community(), channel_id) + .await?; + let members = state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await?; let relay_pubkey_hex = hex::encode(state.relay_keypair.public_key().to_bytes()); let group_id = channel_id.to_string(); @@ -1376,7 +1385,7 @@ async fn handle_put_user( .map_err(|_| anyhow::anyhow!("invalid role: {role_str}"))?, None => state .db - .get_members(tenant.community(), channel_id) + .get_members_for_event_write(tenant.community(), channel_id) .await? .iter() .find(|m| m.pubkey == target_pubkey) @@ -1446,7 +1455,10 @@ async fn handle_remove_user( // Guard: prevent last-owner orphaning on self-removal (kind 9001). if target_pubkey == actor_bytes { - let members = state.db.get_members(tenant.community(), channel_id).await?; + let members = state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await?; let owner_count = members.iter().filter(|m| m.role == "owner").count(); let actor_is_owner = members .iter() @@ -1581,7 +1593,7 @@ async fn handle_edit_metadata( "visibility" => { let was_open = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map(|c| c.visibility == "open") .unwrap_or(false); @@ -1701,8 +1713,10 @@ async fn handle_edit_metadata( // same channel by the same actor could collide ids and skip a fan-out. // Not reachable in practice — unarchive has a single human-driven caller; // the reaper only auto-archives — so we don't engineer around it. - for member in - state.db.get_members(tenant.community(), channel_id).await? + for member in state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await? { if let Err(e) = emit_membership_notification( tenant, @@ -1770,7 +1784,7 @@ async fn handle_delete_event_side_effect( // by sending h=A, e=. if let Some(target_event) = state .db - .get_event_by_id_including_deleted(tenant.community(), &target_id) + .get_event_by_id_including_deleted_for_event_write(tenant.community(), &target_id) .await .map_err(|e| anyhow::anyhow!("get_event_by_id failed: {e}"))? { @@ -1872,7 +1886,11 @@ async fn handle_create_group( // no-h-tag path, ingest never creates the channel, so this is the sole // increment. let channel = if let Some(client_uuid) = extract_h_tag_channel(event) { - match state.db.get_channel(tenant.community(), client_uuid).await { + match state + .db + .get_channel_for_event_write(tenant.community(), client_uuid) + .await + { Ok(ch) => ch, Err(_) => { // Channel not found — shouldn't happen (ingest_event pre-created it), @@ -2026,7 +2044,7 @@ async fn handle_join_request( // Only open channels allow self-join via kind:9021. let channel = state .db - .get_channel(tenant.community(), channel_id) + .get_channel_for_event_write(tenant.community(), channel_id) .await .map_err(|_| anyhow::anyhow!("channel not found"))?; if channel.visibility != "open" { @@ -2104,7 +2122,10 @@ async fn handle_leave_request( let actor_bytes = event.pubkey.to_bytes().to_vec(); // Guard: prevent last-owner orphaning on leave. - let members = state.db.get_members(tenant.community(), channel_id).await?; + let members = state + .db + .get_members_for_event_write(tenant.community(), channel_id) + .await?; let owner_count = members.iter().filter(|m| m.role == "owner").count(); let actor_is_owner = members .iter() @@ -2315,7 +2336,7 @@ async fn handle_standard_deletion_event( for target_id in target_ids { let target_event = match state .db - .get_event_by_id_including_deleted(tenant.community(), &target_id) + .get_event_by_id_including_deleted_for_event_write(tenant.community(), &target_id) .await? { Some(target) => target, @@ -2393,7 +2414,7 @@ async fn handle_standard_deletion_event( if let Ok(react_target_id) = hex::decode(&react_target_hex) { if let Ok(Some(react_target_event)) = state .db - .get_event_by_id(tenant.community(), &react_target_id) + .get_event_by_id_for_event_write(tenant.community(), &react_target_id) .await { let react_target_ts = chrono::DateTime::from_timestamp( @@ -3030,22 +3051,60 @@ async fn emit_initial_ref_state( /// safe to run at startup and periodically without producing an event stream /// when nothing changed. A failure in one community is logged and counted but /// does not prevent the remaining communities from being repaired. +#[derive(Clone, Copy)] +pub enum Nip43ReconciliationPurpose { + /// Before listener admission opens. + Bootstrap, + /// Periodic background repair after startup. + Maintenance, +} + +/// Preserve the original maintenance reconciliation API for downstream callers. +#[deprecated(note = "use reconcile_nip43_membership_snapshots_with_purpose")] pub async fn reconcile_nip43_membership_snapshots(state: &Arc) -> anyhow::Result { - let communities = state.db.usage_community_hosts().await?; + reconcile_nip43_membership_snapshots_with_purpose( + state, + Nip43ReconciliationPurpose::Maintenance, + ) + .await +} + +/// Reconcile NIP-43 snapshots with explicit startup or maintenance attribution. +pub async fn reconcile_nip43_membership_snapshots_with_purpose( + state: &Arc, + purpose: Nip43ReconciliationPurpose, +) -> anyhow::Result { + let communities = match purpose { + Nip43ReconciliationPurpose::Bootstrap => state.db.bootstrap_community_hosts().await?, + Nip43ReconciliationPurpose::Maintenance => state.db.usage_community_hosts().await?, + }; let mut reconciled = 0usize; for community in communities { let community_id = buzz_core::CommunityId::from_uuid(community.id); let host = community.host; let result = async { - if !state - .db - .nip43_membership_snapshot_needs_reconciliation( - community_id, - &state.relay_keypair.public_key(), - ) - .await? - { + let needs_reconciliation = match purpose { + Nip43ReconciliationPurpose::Bootstrap => { + state + .db + .nip43_membership_snapshot_needs_reconciliation_for_bootstrap( + community_id, + &state.relay_keypair.public_key(), + ) + .await? + } + Nip43ReconciliationPurpose::Maintenance => { + state + .db + .nip43_membership_snapshot_needs_reconciliation_for_maintenance( + community_id, + &state.relay_keypair.public_key(), + ) + .await? + } + }; + if !needs_reconciliation { return Ok::(false); } @@ -3270,7 +3329,10 @@ pub async fn reconcile_channel_events( ) -> anyhow::Result<()> { use buzz_db::event::EventQuery; - let channels = state.db.list_channels(tenant.community(), None).await?; + let channels = state + .db + .list_channels_for_bootstrap(tenant.community(), None) + .await?; if channels.is_empty() { return Ok(()); } @@ -3281,7 +3343,7 @@ pub async fn reconcile_channel_events( let channel_id_str = channel.id.to_string(); let existing = match state .db - .query_events(&EventQuery { + .query_events_for_bootstrap(&EventQuery { kinds: Some(vec![39000]), d_tag: Some(channel_id_str.clone()), limit: Some(1), @@ -3414,7 +3476,7 @@ pub async fn publish_nipia_archival_list( let now = nostr::Timestamp::now().as_secs(); let previous = state .db - .query_events(&buzz_db::event::EventQuery { + .query_events_for_event_write(&buzz_db::event::EventQuery { kinds: Some(vec![KIND_IA_ARCHIVED_LIST as i32]), pubkey: Some(relay_pubkey.to_bytes().to_vec()), limit: Some(1), @@ -3517,7 +3579,7 @@ pub async fn publish_dm_visibility_snapshot( let ts = { let existing = state .db - .query_events(&buzz_db::event::EventQuery { + .query_events_for_event_write(&buzz_db::event::EventQuery { kinds: Some(vec![KIND_DM_VISIBILITY as i32]), pubkey: Some(state.relay_keypair.public_key().to_bytes().to_vec()), d_tag: Some(viewer_hex.clone()), @@ -3684,6 +3746,16 @@ pub async fn publish_nipia_unarchived( mod tests { use super::*; + #[test] + fn nip43_reconciliation_compatibility_alias_is_preserved() { + #[allow(deprecated)] + async fn call(state: &Arc) -> anyhow::Result { + reconcile_nip43_membership_snapshots(state).await + } + + let _ = call; + } + #[test] fn group_members_snapshot_keeps_members_past_one_thousand() { let channel_id = Uuid::new_v4(); diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 260dfaed68b..933756aa106 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -302,7 +302,7 @@ async fn main() -> anyhow::Result<()> { ); None } else { - match db.ensure_configured_community(&host).await { + match db.ensure_configured_community_for_bootstrap(&host).await { Ok(record) => { info!(host = %record.host, community = %record.id, "Deployment community ensured"); Some(record.id) @@ -563,7 +563,11 @@ async fn main() -> anyhow::Result<()> { // this repairs pre-snapshot communities and any publication that failed // after a membership transaction committed. if config.require_relay_membership { - match buzz_relay::handlers::side_effects::reconcile_nip43_membership_snapshots(&state).await + match buzz_relay::handlers::side_effects::reconcile_nip43_membership_snapshots_with_purpose( + &state, + buzz_relay::handlers::side_effects::Nip43ReconciliationPurpose::Bootstrap, + ) + .await { Ok(count) => info!(count, "NIP-43 membership snapshots reconciled on startup"), Err(error) => { @@ -582,8 +586,9 @@ async fn main() -> anyhow::Result<()> { interval.tick().await; loop { interval.tick().await; - match buzz_relay::handlers::side_effects::reconcile_nip43_membership_snapshots( + match buzz_relay::handlers::side_effects::reconcile_nip43_membership_snapshots_with_purpose( &reconcile_state, + buzz_relay::handlers::side_effects::Nip43ReconciliationPurpose::Maintenance, ) .await { @@ -1041,6 +1046,7 @@ async fn main() -> anyhow::Result<()> { metrics::gauge!("buzz_db_pool_idle").set(db_stats.idle as f64); metrics::gauge!("buzz_db_pool_active").set(active as f64); metrics::gauge!("buzz_db_pool_max").set(db_stats.max as f64); + pool_state.db.refresh_pool_waiter_metrics(); if let Some(read_stats) = pool_state.db.read_pool_stats() { let read_active = read_stats.size.saturating_sub(read_stats.idle); diff --git a/crates/buzz-relay/src/metrics.rs b/crates/buzz-relay/src/metrics.rs index fb484b01742..0c4bfd2c31f 100644 --- a/crates/buzz-relay/src/metrics.rs +++ b/crates/buzz-relay/src/metrics.rs @@ -37,6 +37,12 @@ const READINESS_DURATION_BUCKETS_S: [f64; 15] = [ 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, ]; +/// Pool checkout buckets: dense around normal sub-100ms waits, with explicit +/// coverage of the reader's 150ms and writer's default three-second budgets. +const DB_POOL_ACQUIRE_DURATION_BUCKETS_S: [f64; 9] = + [0.001, 0.005, 0.01, 0.025, 0.05, 0.15, 0.5, 1.0, 3.0]; +const DB_POOL_ACQUIRE_DURATION_UNIT: metrics::Unit = metrics::Unit::Seconds; + /// Seconds-scale buckets for Git hydration and pack streams. const GIT_DURATION_BUCKETS_S: [f64; 13] = [ 0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, @@ -104,6 +110,11 @@ fn configured_prometheus_builder(gauge_idle_timeout_secs: u64) -> PrometheusBuil &READINESS_DURATION_BUCKETS_S, ) .expect("valid readiness duration bucket boundaries") + .set_buckets_for_metric( + Matcher::Full("buzz_db_pool_acquire_duration_seconds".to_owned()), + &DB_POOL_ACQUIRE_DURATION_BUCKETS_S, + ) + .expect("valid DB pool acquisition duration bucket boundaries") .set_buckets_for_metric( Matcher::Full("buzz_git_hydrate_bytes".to_owned()), &GIT_BYTES_BUCKETS, @@ -158,6 +169,7 @@ pub fn install(port: u16, gauge_idle_timeout_secs: u64) { metrics::set_global_recorder(recorder).expect("global recorder must be set exactly once"); describe_readiness_metrics(); + describe_db_pool_metrics(); tokio::spawn(exporter); } @@ -182,6 +194,23 @@ pub(crate) fn describe_readiness_metrics() { ); } +/// Register the frozen operation-aware pool-acquisition contract. +pub(crate) fn describe_db_pool_metrics() { + metrics::describe_histogram!( + "buzz_db_pool_acquire_duration_seconds", + DB_POOL_ACQUIRE_DURATION_UNIT, + "Database pool checkout duration by valid pool role and operation" + ); + metrics::describe_counter!( + "buzz_db_pool_acquire_attempts_total", + "Database pool checkout terminals by valid pool role, operation, and outcome" + ); + metrics::describe_gauge!( + "buzz_db_pool_waiters", + "Current tracked-operation database pool checkout attempts in progress by valid pool role and operation" + ); +} + #[cfg(test)] pub(crate) fn readiness_test_recorder() -> ( metrics_exporter_prometheus::PrometheusRecorder, @@ -251,3 +280,110 @@ pub async fn track_metrics(req: Request, next: Next) -> Response { response } + +#[cfg(test)] +mod contract_tests { + use std::collections::BTreeSet; + + const OUTCOMES: [&str; 4] = ["success", "timeout", "error", "cancelled"]; + + fn label_keys(line: &str) -> BTreeSet<&str> { + line.split_once('{') + .and_then(|(_, rest)| rest.split_once('}')) + .map(|(labels, _)| { + labels + .split(',') + .filter_map(|label| label.split_once('=').map(|(key, _)| key)) + .collect() + }) + .unwrap_or_default() + } + + #[test] + fn production_builder_exports_frozen_db_pool_contract_and_187_series_budget() { + let (recorder, handle) = super::readiness_test_recorder(); + metrics::with_local_recorder(&recorder, || { + super::describe_db_pool_metrics(); + for (pool_role, operation) in buzz_db::DB_POOL_ACQUIRE_VALID_PAIRS { + metrics::histogram!( + "buzz_db_pool_acquire_duration_seconds", + "pool_role" => pool_role, + "operation" => operation, + ) + .record(0.02); + metrics::gauge!( + "buzz_db_pool_waiters", + "pool_role" => pool_role, + "operation" => operation, + ) + .set(0.0); + for outcome in OUTCOMES { + metrics::counter!( + "buzz_db_pool_acquire_attempts_total", + "pool_role" => pool_role, + "operation" => operation, + "outcome" => outcome, + ) + .increment(1); + } + } + }); + + let scrape = handle.render(); + assert!(scrape.contains("# TYPE buzz_db_pool_acquire_duration_seconds histogram")); + assert!(scrape.contains("# TYPE buzz_db_pool_acquire_attempts_total counter")); + assert!(scrape.contains("# TYPE buzz_db_pool_waiters gauge")); + assert!(scrape.contains("# HELP buzz_db_pool_acquire_duration_seconds Database pool checkout duration by valid pool role and operation")); + assert!(scrape.contains("# HELP buzz_db_pool_acquire_attempts_total Database pool checkout terminals by valid pool role, operation, and outcome")); + assert!(scrape.contains("# HELP buzz_db_pool_waiters Current tracked-operation database pool checkout attempts in progress by valid pool role and operation")); + assert_eq!(super::DB_POOL_ACQUIRE_DURATION_UNIT, metrics::Unit::Seconds); + let readiness_buckets = scrape + .lines() + .filter(|line| { + line.starts_with("buzz_db_pool_acquire_duration_seconds_bucket{") + && line.contains("pool_role=\"writer\"") + && line.contains("operation=\"readiness\"") + }) + .map(|line| { + line.split(",le=\"") + .nth(1) + .and_then(|rest| rest.split_once('"').map(|(bucket, _)| bucket)) + .expect("duration bucket carries le label") + }) + .collect::>(); + assert_eq!( + readiness_buckets, + ["0.001", "0.005", "0.01", "0.025", "0.05", "0.15", "0.5", "1", "3", "+Inf",], + "duration bucket contract drifted:\n{scrape}" + ); + + let raw_series = scrape + .lines() + .filter(|line| { + line.starts_with("buzz_db_pool_acquire_duration_seconds") + || line.starts_with("buzz_db_pool_acquire_attempts_total") + || line.starts_with("buzz_db_pool_waiters{") + }) + .collect::>(); + assert_eq!( + raw_series.len(), + buzz_db::DB_POOL_ACQUIRE_RAW_SERIES_PER_POD, + "unexpected raw scrape:\n{scrape}" + ); + + for line in raw_series { + let keys = label_keys(line); + if line.starts_with("buzz_db_pool_acquire_duration_seconds_bucket") { + assert_eq!(keys, BTreeSet::from(["le", "operation", "pool_role"])); + } else if line.starts_with("buzz_db_pool_acquire_duration_seconds") { + assert_eq!(keys, BTreeSet::from(["operation", "pool_role"])); + } else if line.starts_with("buzz_db_pool_acquire_attempts_total") { + assert_eq!(keys, BTreeSet::from(["operation", "outcome", "pool_role"])); + } else { + assert_eq!(keys, BTreeSet::from(["operation", "pool_role"])); + } + assert!(!line.contains("operation=\"other\"")); + assert!(!line.contains("result=")); + } + } +} diff --git a/crates/buzz-relay/src/readiness.rs b/crates/buzz-relay/src/readiness.rs index 36a36c228b2..79a1a985707 100644 --- a/crates/buzz-relay/src/readiness.rs +++ b/crates/buzz-relay/src/readiness.rs @@ -8,7 +8,7 @@ use std::future::Future; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Duration; -use buzz_db::{Db, DbReadinessOutcome}; +use buzz_db::{Db, DbError, DbReadinessOutcome}; use tokio::time::Instant; const READINESS_TIMEOUT: Duration = Duration::from_secs(2); @@ -327,13 +327,20 @@ async fn redis_check(pool: &deadpool_redis::Pool, deadline: Instant) -> RedisOut } async fn deletion_catalog_check(db: &Db, deadline: Instant) -> DeletionCatalogOutcome { - match tokio::time::timeout_at(deadline, db.validate_deletion_serving_catalog()).await { - Err(_) => DeletionCatalogOutcome::OperationTimeout, - Ok(Err(error)) => { + classify_deletion_catalog_result( + db.validate_deletion_serving_catalog_for_readiness(deadline) + .await, + ) +} + +fn classify_deletion_catalog_result(result: buzz_db::Result<()>) -> DeletionCatalogOutcome { + match result { + Err(DbError::Sqlx(sqlx::Error::PoolTimedOut)) => DeletionCatalogOutcome::OperationTimeout, + Err(error) => { tracing::debug!(error = %error, "Deletion catalog readiness validation failed"); DeletionCatalogOutcome::OperationError } - Ok(Ok(())) => DeletionCatalogOutcome::Success, + Ok(()) => DeletionCatalogOutcome::Success, } } @@ -692,6 +699,18 @@ mod tests { assert_eq!(READINESS_RAW_SERIES_PER_POD, 99); } + #[test] + fn deletion_catalog_deadline_is_a_timeout_not_an_operation_error() { + assert_eq!( + classify_deletion_catalog_result(Err(DbError::Sqlx(sqlx::Error::PoolTimedOut))), + DeletionCatalogOutcome::OperationTimeout + ); + assert_eq!( + classify_deletion_catalog_result(Err(DbError::InvalidData("catalog".into()))), + DeletionCatalogOutcome::OperationError + ); + } + #[test] fn slow_older_failure_cannot_overwrite_newer_success_gauges() { let coordinator = ReadinessCoordinator::default(); diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index 47665db6779..bf51a2ff3af 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -1244,7 +1244,7 @@ impl AppState { pub async fn revalidate_live_communities(&self) -> usize { let (closed, failures) = revalidate_registered_communities(&self.community_connections, |community_id| { - self.db.is_community_active(community_id) + self.db.is_community_active_for_maintenance(community_id) }) .await; for (community_id, error) in failures { diff --git a/crates/buzz-relay/src/workflow_sink.rs b/crates/buzz-relay/src/workflow_sink.rs index 17016ca0d84..6450b15b282 100644 --- a/crates/buzz-relay/src/workflow_sink.rs +++ b/crates/buzz-relay/src/workflow_sink.rs @@ -257,7 +257,7 @@ impl ActionSink for RelayActionSink { let channel = state .db - .get_channel(tenant.community(), channel_uuid) + .get_channel_for_event_write(tenant.community(), channel_uuid) .await .map_err(|e| match &e { buzz_db::DbError::ChannelNotFound(_) | buzz_db::DbError::NotFound(_) => { @@ -364,13 +364,13 @@ impl ActionSink for RelayActionSink { // must not drop the message, so log and proceed with the base tags. let members = state .db - .get_members(tenant.community(), channel_uuid) + .get_members_for_event_write(tenant.community(), channel_uuid) .await .map_err(|e| ActionSinkError::Database(e.to_string()))?; let member_pubkeys: Vec> = members.iter().map(|m| m.pubkey.clone()).collect(); let users = state .db - .get_users_bulk(tenant.community(), &member_pubkeys) + .get_users_bulk_for_event_write(tenant.community(), &member_pubkeys) .await .map_err(|e| ActionSinkError::Database(e.to_string()))?; let named_members: Vec<(String, String)> = users @@ -994,7 +994,7 @@ mod postgres_tests { .to_vec(); state .db - .get_event_by_id(community, &id_bytes) + .get_event_by_id_for_event_write(community, &id_bytes) .await .expect("query event") .expect("event persisted") @@ -1117,7 +1117,7 @@ mod postgres_tests { .to_vec(); let stored = state .db - .get_event_by_id(community, &reply_id_bytes) + .get_event_by_id_for_event_write(community, &reply_id_bytes) .await .expect("query reply") .expect("reply persisted"); @@ -1286,7 +1286,7 @@ mod postgres_tests { // reply→the immediate parent (matching the ingest resolver). let stored = state .db - .get_event_by_id(community, &reply_id_bytes) + .get_event_by_id_for_event_write(community, &reply_id_bytes) .await .expect("query reply") .expect("reply persisted"); diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index b022dd8a27f..a4545827bca 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -136,6 +136,48 @@ Shutdown without dependency evaluation increments only `buzz_readiness_checks_total{reason="shutting_down"}` and sets the overall state to zero; it does not fabricate dependency failures or latency samples. +### Operation-aware database pool acquisition contract + +The operation-aware families separate three questions: who is waiting now, +how completed/abandoned attempts ended, and how long checkout waits took. +Outcome remains on the terminal counter for historical deployment comparison; +it is intentionally absent from the expensive duration histogram. + +These families cover the explicitly routed deployment-critical operations +listed below; they are not a count of every SQLx checkout in Buzz. In +particular, a zero operation waiter does not prove that the shared SQLx pool +has no uninstrumented waiter. Interpret it beside the pool active, idle, and +maximum gauges when diagnosing total capacity pressure. + +| Metric | Type | Labels | +|--------|------|--------| +| `buzz_db_pool_acquire_duration_seconds` | histogram | `pool_role`, `operation` | +| `buzz_db_pool_acquire_attempts_total` | counter | `pool_role`, `operation`, `outcome` | +| `buzz_db_pool_waiters` | gauge | `pool_role`, `operation`; tracked operations only, periodically refreshed including zero | + +Outcomes are `success`, `timeout`, `error`, and `cancelled`. Operations are +`bootstrap`, `readiness`, `tenant_resolution`, `authentication`, +`authorization`, `subscription_history`, `event_write`, and `maintenance`. +Only the following eleven pairs are valid: + +```text +writer/bootstrap reader/bootstrap +writer/readiness +writer/tenant_resolution +writer/authentication +writer/authorization reader/authorization +writer/subscription_history reader/subscription_history +writer/event_write +writer/maintenance +``` + +Nine finite checkout buckets plus `+Inf`, sum, and count yield 12 histogram +series per valid pair. The new contract therefore has a hard ceiling of 187 +raw Prometheus series per pod: `11 × (12 + 4 + 1)`. The two legacy acquisition +families remain temporarily for dashboard compatibility and are not part of +that new-family budget. No `other` operation or request-controlled/sensitive +label is valid. + ## Relay Pod extensions The chart exposes narrow extension points for init containers, volumes, relay