From 73e1683be248e95e515628d4227c2a30125bcb7b Mon Sep 17 00:00:00 2001 From: Cosmin Staicu Date: Tue, 6 Oct 2026 23:04:47 +0300 Subject: [PATCH] perf(cache)!: share concurrent inner reads and serve an outage from the local tier On a local miss, concurrent reads of one key now wait for a single inner-tier read and share its result or failure: GetAsync, GetCacheEntryAsync, the hash reads, and the read GetOrAddAsync makes before its lock. Reads share only when their key, type and local lifetimes match. A caller that cancels stops waiting; the shared read is cancelled once every caller waiting on it has cancelled, and its local copy is committed only while a caller still waits. When the inner tier refuses a GetOrAddAsync write, single or batch, the generated value is kept locally for LocalMaxExpirationDisconnected, so the callers waiting on the local lock reuse it instead of each running the generator again. CacheOptions.ConnectionMonitorEnabled and UseLocalOnlyWhenDisconnected default to true, so an outage serves and keeps values locally, and a hit read while the tier is down is kept for the disconnected cap. The new ClearLocalOnReconnect, true by default, clears the local tier once a Redis Pub/Sub broadcast is connected again, since invalidations published meanwhile never arrived. Redis Streams replay them, so their local tier is kept. RedisConnector.IsConnected is false only when a multiplexer exists and reports itself down, so a cold connector still sends its first command. A Redis Streams consumer now checks before each read whether entries past its last delivery were trimmed, and treats a group or stream removed while in use as a loss too. The check and the read run in one script, so a trim cannot land between them, and a gap is reported once however often the stream is polled before the group reads past it. On Redis 7.0 and later the group is created with the count of entries before it (ENTRIESREAD), so a trim before its first read shows and a history trimmed before it existed does not. Where Redis cannot tell the count, the one last found at the same position stands in; with none, a trim past the last delivery counts as a loss. Either way, every local entry on that topic expires, in each cache that uses it, through the new IEventSubject.Invalidate(MissedEventsReason). A Pub/Sub topic expires what it kept before its delayed subscription went in place, at first and after a reconnect, unless the cache turned ClearLocalOnReconnect off. Multi-key reads stored each hit locally as a key-value pair, so no later read found it; they now keep the value. A refresh is broadcast as CacheRefreshed; it was sent as CacheRemoved. A removal or a loss after a refresh expires the local entry instead of re-inserting the refreshed value, and a refresh delivered after it no longer brings the value back. An event dropped at a full dispatcher channel expires the topic's local entries. Closes #222. BREAKING CHANGE: a disconnected inner tier now serves the local copy instead of answering misses, and the connection monitor is on by default. Set UseLocalOnlyWhenDisconnected or ConnectionMonitorEnabled to false for the previous behavior. IEventSubject gains Invalidate(MissedEventsReason); a custom subject passed to a topic constructor must implement it, calling the new public IMissedEventsObserver on its observers. IMultilayerCacheOptions gains ClearLocalOnReconnect, which a custom implementation must add. Signed-off-by: Cosmin Staicu --- CHANGELOG.md | 71 +++ docs/concepts.md | 43 +- docs/recipes/second-redis-connection.md | 2 +- docs/reference/interfaces.md | 4 +- docs/reference/settings.md | 14 +- .../appsettings.all.json | 12 +- .../CacheOptions.cs | 2 +- .../IConnectionState.cs | 2 +- src/UiPath.Caching/Broadcast/ChangeToken.cs | 70 ++- .../Broadcast/ChangeTokenFactory.cs | 8 +- src/UiPath.Caching/Broadcast/ChannelHelper.cs | 26 +- src/UiPath.Caching/Broadcast/DroppedEvents.cs | 58 ++ .../Broadcast/GapFilteringTopic.cs | 52 ++ src/UiPath.Caching/Broadcast/IEventSubject.cs | 3 + .../Broadcast/IMaskedChangeTokenFactory.cs | 2 +- .../Broadcast/IMissedEventsObserver.cs | 11 + src/UiPath.Caching/Broadcast/KeyedSubject.cs | 76 ++- .../Broadcast/MissedEventsReason.cs | 11 + .../Broadcast/Redis/ConsumerGroups.cs | 80 +++ .../Broadcast/Redis/ReadCount.cs | 4 + .../Redis/RedisPubSubSubjectWriter.cs | 152 ++++- .../Broadcast/Redis/RedisPubSubTopic.cs | 37 +- .../Redis/RedisStreamSubjectWriter.cs | 221 ++++++- .../Broadcast/Redis/RedisStreamsTopic.cs | 39 +- .../Broadcast/Redis/StreamGap.cs | 7 + .../Broadcast/Redis/StreamIds.cs | 125 ++++ src/UiPath.Caching/CacheEventPublisher.cs | 2 +- src/UiPath.Caching/HashLocalMemorySetter.cs | 5 +- src/UiPath.Caching/IInFlightRun.cs | 10 + src/UiPath.Caching/IMultilayerCacheOptions.cs | 5 +- src/UiPath.Caching/InFlight.cs | 164 ++++++ src/UiPath.Caching/InFlightKey.cs | 4 + src/UiPath.Caching/InMemoryCacheOptions.cs | 2 + .../InMemoryRedisCacheOptions.cs | 2 + src/UiPath.Caching/LocalMemorySetter.cs | 5 +- src/UiPath.Caching/MemoryCacheSetter.cs | 13 +- src/UiPath.Caching/MultilayerCache.cs | 122 ++-- src/UiPath.Caching/MultilayerCacheBase.cs | 32 +- src/UiPath.Caching/MultilayerHashCache.cs | 56 +- src/UiPath.Caching/PublicAPI.Unshipped.txt | 12 + .../Redis/ConnectionStateMonitor.cs | 174 +++++- src/UiPath.Caching/Redis/RedisConnector.cs | 8 +- .../Broadcast/ChangeTokenTests.cs | 159 +++++ .../Broadcast/ChannelHelperTests.cs | 15 + .../Broadcast/DroppedEventsTests.cs | 115 ++++ .../Broadcast/EventDispatcherTests.cs | 1 + .../Broadcast/KeyedSubjectTests.cs | 86 ++- .../RedisPubSubSubjectWriterTests.cs | 187 ++++++ .../Broadcast/RedisPubSubTopicTests.cs | 103 ++++ .../RedisStreamGapIntegrationTests.cs | 362 ++++++++++++ .../RedisStreamSubjectWriterTests.cs | 305 +++++++++- .../Broadcast/RedisStreamsTopicTests.cs | 97 ++++ .../Broadcast/StreamIdsTests.cs | 100 ++++ .../CacheEventPublisherTests.cs | 37 ++ .../Fakes/InMemoryMultilayer.cs | 11 +- tests/UiPath.Caching.Tests/InFlightTests.cs | 234 ++++++++ .../InnerTierOutageTests.cs | 542 ++++++++++++++++++ ...MultilayerCachePerNamePolicyWiringTests.cs | 2 + .../MultilayerCacheTests.cs | 5 +- .../MultilayerHashCachePerNameJitterTests.cs | 2 + .../MultilayerHashCacheRehydrateTests.cs | 2 + .../MultilayerHashCacheTests.cs | 7 +- .../ConnectionStateMonitorRecoveredTests.cs | 437 ++++++++++++++ .../Redis/RedisConnectorLifecycleTests.cs | 19 +- .../UiPath.Caching.Tests/SpanKeyReadTests.cs | 4 +- 65 files changed, 4395 insertions(+), 185 deletions(-) create mode 100644 src/UiPath.Caching/Broadcast/DroppedEvents.cs create mode 100644 src/UiPath.Caching/Broadcast/GapFilteringTopic.cs create mode 100644 src/UiPath.Caching/Broadcast/IMissedEventsObserver.cs create mode 100644 src/UiPath.Caching/Broadcast/MissedEventsReason.cs create mode 100644 src/UiPath.Caching/Broadcast/Redis/ConsumerGroups.cs create mode 100644 src/UiPath.Caching/Broadcast/Redis/ReadCount.cs create mode 100644 src/UiPath.Caching/Broadcast/Redis/StreamGap.cs create mode 100644 src/UiPath.Caching/Broadcast/Redis/StreamIds.cs create mode 100644 src/UiPath.Caching/IInFlightRun.cs create mode 100644 src/UiPath.Caching/InFlight.cs create mode 100644 src/UiPath.Caching/InFlightKey.cs create mode 100644 tests/UiPath.Caching.Tests/Broadcast/DroppedEventsTests.cs create mode 100644 tests/UiPath.Caching.Tests/Broadcast/RedisStreamGapIntegrationTests.cs create mode 100644 tests/UiPath.Caching.Tests/Broadcast/StreamIdsTests.cs create mode 100644 tests/UiPath.Caching.Tests/CacheEventPublisherTests.cs create mode 100644 tests/UiPath.Caching.Tests/InFlightTests.cs create mode 100644 tests/UiPath.Caching.Tests/InnerTierOutageTests.cs create mode 100644 tests/UiPath.Caching.Tests/Redis/ConnectionStateMonitorRecoveredTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1149f..f37cd79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,38 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Changed +- **A Redis outage is served from the local tier by default.** `CacheOptions.ConnectionMonitorEnabled` and + `UseLocalOnlyWhenDisconnected` now default to `true`: while the inner tier is disconnected, reads are served from + the local tier and writes kept there, capped at `LocalMaxExpirationDisconnected` (30 seconds by default). The new + `ClearLocalOnReconnect`, also `true` by default, expires the local entries of a Redis Pub/Sub topic when its + connection is restored, since the invalidations sent while it was down never arrived. The topic expires them through + their change tokens, so it works on any local tier, a custom `IMemoryCacheFactory` one included. A forced reconnect counts too: swapping the connection + can drop publications although it never read as down. A Pub/Sub topic also subscribes only after a delay, at first + and after a reconnect, so once its subscription is in place it expires the entries cached before then; a cache with + `ClearLocalOnReconnect` off keeps them. Redis Streams replay what was sent meanwhile, so their local tier is kept. + The monitor setting is app-wide, so the broadcast providers' connection monitor turns on too. **Breaking** for an app that relied on a disconnected tier answering + misses: set `UseLocalOnlyWhenDisconnected` to `false`, or `ConnectionMonitorEnabled` to `false` for the previous + behavior. **Breaking** too for a custom `IMultilayerCacheOptions` implementation, which must add + `ClearLocalOnReconnect`. + +- **`RedisConnector.IsConnected` is `false` only when a connection exists and reports itself down.** Before the + first connect, while it is pending and after it faulted, it is now `true`, so a command goes through and opens or + retries the connection; it used to be `false`, and the operations that check it first, L2 reads and every + `ISetCache` call, did nothing until a write had connected. Observable to code that reads `IRedisConnector.IsConnected` + directly, such as a status endpoint: use `RedisHealthCheck` to learn whether the server is reachable. + +- **Concurrent single-key reads share one inner read.** On a local miss, concurrent single-key `GetAsync`, + `GetCacheEntryAsync` and hash reads of one key, and the read `GetOrAddAsync` makes before its lock, wait for a + single inner-tier read and share its result or failure. Reads share only when they read the same value type with + the same `LocalExpiration` and `LocalExpirationDisconnected`, since the shared read keeps its hit for those + lifetimes; reads that differ in any of them read separately. Multi-key reads still read every missing key + themselves. A caller that cancels stops waiting; the shared read is cancelled only once every caller waiting on it + has cancelled. + +- **`GetOrAddAsync` keeps a value the inner tier refused.** When the inner write fails or throws, the generated value + is kept in the local tier for `LocalMaxExpirationDisconnected`, by the batch overload too, so the callers waiting on + the local lock reuse it instead of each running the generator again. + - **A string key reads the local tier by its text.** On .NET 9 and later, `GetAsync`, `GetItemAsync`, `ContainsAsync`, `GetCacheEntryAsync` and `GetOrAddAsync` with a `CacheKey` look the in-memory tier up as the span reads do, and `Cache` and `HashCache` compose their strategy's key on the stack instead of through @@ -125,6 +157,45 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Fixed +- **Events dropped at a full dispatcher channel expire what the topic keeps.** A Redis Pub/Sub topic whose channel + was full in `Wait` mode discarded the event, and the drop modes (`DropOldest`, `DropNewest`, `DropWrite`) discarded + one silently on either transport; the invalidation it carried was lost. Each drop now expires the topic's local + entries, coalesced to one expiry at a time. + +- **A removal after a refresh expires the local entry.** A change token kept the metadata and expiration of an earlier + refresh, so a later removal or loss, which carries neither, re-inserted the old value under a fresh token. It now + clears them first, and a refresh delivered after the removal or loss no longer brings them back. + +- **A refresh is broadcast as `CacheRefreshed`.** `CacheEventPublisher.CacheRefreshedAsync`, which `RefreshAsync` + raises, sent the `CacheRemoved` type. The library's own receivers treat both alike, so only a subscriber that + filters by event type sees the difference. + +- **A Redis Streams topic that loses entries before this node reads them expires what it kept.** Before each read + the consumer checks whether entries past its last delivery were trimmed (`MaxLength`, or the maintainer's trim) or deleted, + since a trim can reach unread entries between any two polls, and a consumer group or stream removed while in use + counts as a loss too. The check and the read run in one script, so a trim cannot land between them; where scripts + are refused, they run apart, and where `XINFO` is denied, each reconnect counts as a loss. A gap is reported + once: until the group reads past it or more entries are trimmed, later checks find the same one. The local entries on that topic then + expire, in every cache that uses it, whatever `ClearLocalOnReconnect` says. They used to be kept, so a change + whose invalidation was trimmed was served stale until the entry expired. Below Redis 7.0, which does not count + the entries a group read, any trim past this node's last delivery counts as a loss, even one that removed only + entries this node had read. From 7.0, the consumer group is created with the count of entries before it + (`ENTRIESREAD`), so a trim before its first read shows and entries trimmed before it existed do not count. + Where Redis cannot tell the count, as after a read past a deleted entry, the count last found at the same + position stands in; with none found, a trim past the last delivery counts as a loss. The topic + signals the loss through the new + `IEventSubject.Invalidate(MissedEventsReason)`: `Lost` for a Streams loss, which every cache honors, and + `SubscriptionGap` for the Pub/Sub delay above. **Breaking** for a custom subject passed to the `RedisPubSubTopic` or + `RedisStreamsTopic` constructor: implement `Invalidate` to expire what its observers keep. The library's change + tokens implement the new public `IMissedEventsObserver`, so such a subject calls `OnEventsMissed(reason)` on each + observer that implements it. A custom `IChangeTokenFactory` needs no change: for a cache that keeps its entries + over a subscription gap, it is handed a view of the topic whose observers do not hear the gap. + +- **Multi-key reads keep a usable local copy.** `GetAsync` and `GetCacheEntriesAsync` over several keys stored each + inner-tier hit locally as a key-value pair rather than as the value, so no later read found it and every one went + back to the inner tier. A hit is now kept as a single-key read keeps it, for `LocalMaxExpirationDisconnected` while + the inner tier is disconnected. + - **The key prefix no longer depends on `RedisCacheOptions.KeyPrefix` being set.** A connector whose `IDatabase` is wrapped with `WithKeyPrefix` stores every key under that prefix, and two places need it: the stream maintainer, to scan for and strip it, and the cluster slot check, to hash the key the server receives. Both relied on `KeyPrefix` diff --git a/docs/concepts.md b/docs/concepts.md index ed09be2..bb8bfe7 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -138,10 +138,35 @@ expire relatively quickly so that as soon as connectivity is restored the proces re-populates from a fresh L2 rather than serving stale data for the full `LocalMaxExpiration` window. -`UseLocalOnlyWhenDisconnected` changes the disconnected behavior further: when -true, an unhealthy L2 means reads are served from L1 only and no L2 round-trip -is attempted. This is appropriate when you would rather serve potentially stale -data than accumulate timeout latency on every cache miss during a Redis outage. +`UseLocalOnlyWhenDisconnected`, on by default, changes the disconnected behavior +further: while L2 is unhealthy, an L1 hit is served and writes are kept in L1 +only, capped at `LocalMaxExpirationDisconnected`. That serves potentially stale +data rather than losing the local copy during a Redis outage. Set it to `false` to +drop the local entry and answer a miss instead. An L1 miss still asks L2: the +monitor also tracks the broadcast connection, so "unhealthy" can mean only that one +is down, and with the fail-fast backlog policy a read on a dead data connection +fails at once rather than waiting for its timeout. + +`ClearLocalOnReconnect`, also on by default, expires a `RedisPubSub` topic's +L1 entries when its connection is restored, through their change tokens, so on +any local tier. Invalidations published while the connection was down never +reached this node, so an entry cached before the outage could otherwise outlive a +change made during it. `RedisStreams` needs no clear: the consumer group resumes +after its last delivery and replays what the outage held back. When entries past +it were trimmed first, or the group or stream was removed while in use, the topic +expires every local entry it invalidates, whatever `ClearLocalOnReconnect` says. +Below Redis 7.0, which does not count the entries a group read, any trim past this +node's last delivery counts as a loss, even one that removed only entries it had +read. Concurrent single-key reads of one key share a single L2 +read when they read the same value type with the same `LocalExpiration` and +`LocalExpirationDisconnected`, so the refill after a clear costs one read per key +per node for each such combination; reads that differ in any of them, and +multi-key reads, read L2 themselves. + +A `GetOrAddAsync` miss runs its generator under the local lock, and the callers +waiting on the lock read the stored value once it is written. When L2 refuses the +write, the generated value is kept in L1 for `LocalMaxExpirationDisconnected`, so +those waiters reuse it instead of each running the generator again. Why split into two tiers at all? L1 eliminates Redis latency and bandwidth on the hot path — for a busy service, removing the network hop from cache reads can @@ -152,11 +177,11 @@ tradeoff is that each node's L1 can drift from L2 when another node writes a new value. Topics solve that problem, and their relationship to layers is explained in the next section. -It is worth noting that all three of these disconnected-scenario options -(`LocalMaxExpiration`, `LocalMaxExpirationDisconnected`, -`UseLocalOnlyWhenDisconnected`) require the connection monitor to be active — -either by setting `ConnectionMonitorEnabled` to `true` on `CacheOptions`, or by -setting it on `InMemoryRedisCacheOptions` directly. Without the monitor the cache +It is worth noting that these disconnected-scenario options +(`LocalMaxExpirationDisconnected`, `UseLocalOnlyWhenDisconnected`, +`ClearLocalOnReconnect`) require the connection monitor to be active; +`LocalMaxExpiration` caps every L1 entry and does not. It is on by default (`CacheOptions.ConnectionMonitorEnabled`), +and `InMemoryRedisCacheOptions.ConnectionMonitorEnabled` overrides it per provider. Without the monitor the cache cannot distinguish "connected" from "disconnected" and the disconnected behavior never triggers. diff --git a/docs/recipes/second-redis-connection.md b/docs/recipes/second-redis-connection.md index 24bd761..52ebb1f 100644 --- a/docs/recipes/second-redis-connection.md +++ b/docs/recipes/second-redis-connection.md @@ -80,7 +80,7 @@ services.AddNamedCaching("SecondaryRedis", configuration, chain, **A separate section.** When the second server needs different *cache* options too, pass `sectionName`: `AddNamedCaching("cold", configuration, chain, sectionName: "CachingCold")` binds everything from `CachingCold`, with the connection at `CachingCold:Connections:cold`. The two sections must agree on `KeyCasing`, because `AddCaching` seeds the process-wide `CacheKey.DefaultCasing`; a stack that differs is refused, naming both values, whether the casing came from its section or from a `Configure` in `configureServices`. -**Warm-up.** Set `WarmUpOnStart` on the primary connection (the stack inherits it). A connection is otherwise opened by the first *write* through `ICache` or `IHashCache`; the operations that check `IsConnected` first (every `ISetCache` call, L2 reads) do nothing while it is closed and never open it themselves. With warm-up, the stack's hosted services, started with the host, open the second connection at startup, and both servers log a handshake as the app starts. +**Warm-up.** Set `WarmUpOnStart` on the primary connection (the stack inherits it). A connection is otherwise opened by the first command through `ICache`, `IHashCache` or `ISetCache`, read or write: a connector reports itself connected until it has a multiplexer that says otherwise, so the operations that check `IsConnected` first go through and open it. With warm-up, the stack's hosted services, started with the host, open the second connection at startup, and both servers log a handshake as the app starts. **Names.** The name is the DI key and the connection section, so it is case-sensitive as a key. `Redis` is refused, since it names the primary connection, and so is a second `AddNamedCaching` with the same name. diff --git a/docs/reference/interfaces.md b/docs/reference/interfaces.md index a8aafcb..60d331f 100644 --- a/docs/reference/interfaces.md +++ b/docs/reference/interfaces.md @@ -868,11 +868,11 @@ public interface IConnectionState } ``` -A non-blocking snapshot of whether the backing store is reachable, plus the transitions as events. `IsConnected` never blocks and never throws. Implemented by `RedisCacheBase`, so `Redis`-provider caches expose it; the multilayer caches and `NullCache` do not, and neither does `Cache`, which holds its underlying `ICache` privately. +A non-blocking snapshot of whether commands should go to the backing store, plus the transitions as events. `IsConnected` never blocks and never throws. For `RedisConnector` it is `false` only once a connection exists and reports itself down: before the first connect, while it is pending, and after it faulted, it is `true`, so a command goes through and opens or retries the connection. Implemented by `RedisCacheBase`, so `Redis`-provider caches expose it; the multilayer caches and `NullCache` do not, and neither does `Cache`, which holds its underlying `ICache` privately. What it is *not*: a way to explain a negative result. It is a cached snapshot refreshed on connection events and a timer, it says nothing about whether any particular command succeeded, and it is `true` both where there is nothing to disconnect from and where `ConnectionMonitorEnabled` is off. A `false` from `SetAsync` or [`TryAddAsync`](#icache) can perfectly well coincide with `IsConnected == true` — a serialization failure or a rejected command does that — so reading it afterwards does not recover why the call failed. -**Use this when:** you are reporting or reacting to cache *health* — a readiness probe, a metric, a log line, or backing off writes while a tier is known down. Subscribe to `OnConnectionFailed` / `OnConnectionRestored` for the transitions rather than polling. +**Use this when:** you are reacting to a tier known to be down — a metric, a log line, or backing off writes. Subscribe to `OnConnectionFailed` / `OnConnectionRestored` for the transitions rather than polling. For a readiness probe, use `RedisHealthCheck`, which asks the server: `IsConnected` is `true` while a first connection is still failing. ## Telemetry seam diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 23680f0..f5ebc40 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -29,7 +29,7 @@ Every binding-visible property on every shipped options class, with shipped defa | `AppShortName` | `string` | _required_ | App-wide | Short application name prefixed to every cache key; apps throw at startup if blank or missing. | | `KeyCasing` | `CacheKeyCasing` | `Insensitive` | App-wide | Key case folding applied when a key is built without an explicit mode, including every implicit `string` -> `CacheKey` conversion. `Insensitive` trims and lowercases (historical behavior); `Sensitive` preserves the caller's casing. **Changing this relocates every cache key** — existing entries become unreachable and are rewritten under the new spelling. The distributed cache always uses `Sensitive` and is unaffected. Scope is the **process**, not the container: `CacheKey` is a struct built by callers without access to DI, so the setting is seeded into the static `CacheKey.DefaultCasing`. Hosting two differently-configured containers in one process is therefore unsupported — the last `AddCaching` wins for both. | | `LargeValueThreshold` | `int` | `20000` | App-wide | Byte threshold for audit logging; writes whose payload exceeds this are logged when `AuditEnabled` is `true`. | -| `ConnectionMonitorEnabled` | `bool` | `false` | App-wide | Enable Redis health-check polling app-wide; provider-level `ConnectionMonitorEnabled` inherits this when `null`. | +| `ConnectionMonitorEnabled` | `bool` | `true` | App-wide | Enable Redis health-check polling app-wide; provider-level `ConnectionMonitorEnabled` inherits this when `null`. | | `LocalLockPoolSize` | `int` | `100` | App-wide | Semaphore pool size for the default local lock — allocation hint, not a hard concurrency cap. | | `LocalLockPoolInitialFill` | `int` | `10` | App-wide | Semaphores pre-allocated at startup; must be in `[0, LocalLockPoolSize]`. | | `DistributedLockPollInterval` | `TimeSpan` | `00:00:00.050` | App-wide | Initial wait between distributed-lock acquire retries; doubles per attempt up to `DistributedLockMaxPollInterval`. | @@ -175,7 +175,8 @@ Per-topic overrides: add entries to `Topics[]` under `Broadcast:RedisPubSub`. Ea | `ConnectionMonitorPeriod` | `TimeSpan?` | `00:00:05` | Per-provider | How often the connection monitor probes Redis health. | | `SizeLimit` | `long?` | `null` | Per-provider | Max bytes for the in-memory tier; `null` = unlimited. | | `CompactionPercentage` | `double?` | `null` | Per-provider | Fraction of `SizeLimit` to free when the limit is hit; `null` = runtime default (0.05). | -| `UseLocalOnlyWhenDisconnected` | `bool?` | `null` | Per-provider | `null` = `false`; `true` = serve L1-only responses when L2 is disconnected. | +| `UseLocalOnlyWhenDisconnected` | `bool?` | `null` | Per-provider | `null` = `true`: serve and keep L1-only while L2 is disconnected; `false` = drop the local entry and answer a miss. | +| `ClearLocalOnReconnect` | `bool?` | `null` | Per-provider | `null` = `true`: clear L1 when a `RedisPubSub` broadcast reconnects, since invalidations published while it was down never arrived. Inert over `RedisStreams`, which replay them and expire a topic's entries themselves when entries were lost. | | `LocalMaxExpirationDisconnected` | `TimeSpan?` | `00:00:30` | Per-provider | L1 TTL cap while L2 is disconnected; limits the stale-read window. | | `LocalLockEnabled` | `bool?` | `true` | Per-provider | Acquire a local (in-process) lock before calling the value factory. | | `LocalLockTimeout` | `TimeSpan?` | `00:00:00.500` | Per-provider | Max wait to acquire the local lock before bypassing it. | @@ -218,13 +219,14 @@ Per-topic overrides: add entries to `Topics[]` under `Broadcast:RedisPubSub`. Ea | `BroadcastEnable` | `bool` | `false` | Per-provider | Enable broadcast invalidation for this in-memory cache instance. | | `Topic` | `string?` | `null` | Per-provider | Topic name for invalidation broadcasts; `null` = use `CacheOptions.DefaultTopic`. | | `LocalMaxExpiration` | `TimeSpan?` | `01:00:00` | Per-provider | Cap on in-memory TTL; `null` = no cap (falls back to the resolved `DefaultExpiration`). | -| `ConnectionMonitorEnabled` | `bool?` | `null` | Per-provider | Inert for this provider (no Redis connection); present to satisfy `IMultilayerCacheOptions`. | +| `ConnectionMonitorEnabled` | `bool?` | `null` | Per-provider | Applies only when broadcast runs over Redis: `null` = inherit from `CacheOptions.ConnectionMonitorEnabled`; off turns off `UseLocalOnlyWhenDisconnected` and `ClearLocalOnReconnect` too. Inert otherwise. | | `CacheNullValues` | `bool` | `false` | Per-provider | Persist `null`/empty factory returns as sentinels. | -| `ConnectionMonitorPeriod` | `TimeSpan?` | `00:00:05` | Per-provider | Inert for this provider; present to satisfy `IMultilayerCacheOptions`. | +| `ConnectionMonitorPeriod` | `TimeSpan?` | `00:00:05` | Per-provider | Applies only when broadcast runs over Redis: how often the monitor polls the broadcast connection while it reads as down. Inert otherwise. | | `SizeLimit` | `long?` | `null` | Per-provider | Max bytes for the in-memory store; `null` = unlimited. | | `CompactionPercentage` | `double?` | `null` | Per-provider | Fraction of `SizeLimit` to free when the limit is hit; `null` = runtime default (0.05). | -| `UseLocalOnlyWhenDisconnected` | `bool?` | `null` | Per-provider | Inert for this provider; present to satisfy `IMultilayerCacheOptions`. | -| `LocalMaxExpirationDisconnected` | `TimeSpan?` | `00:00:30` | Per-provider | Inert for this provider; present to satisfy `IMultilayerCacheOptions`. | +| `UseLocalOnlyWhenDisconnected` | `bool?` | `null` | Per-provider | Applies only when broadcast runs over Redis: `null` = `true`, keep serving L1 while the broadcast connection is down. Inert otherwise. | +| `ClearLocalOnReconnect` | `bool?` | `null` | Per-provider | Applies only when broadcast runs over `RedisPubSub`: `null` = `true`, clear L1 once the broadcast connection recovers, since invalidations sent meanwhile were missed. Inert otherwise. | +| `LocalMaxExpirationDisconnected` | `TimeSpan?` | `00:00:30` | Per-provider | Applies only when broadcast runs over Redis: the L1 lifetime cap for values written while the broadcast connection is down. Inert otherwise. | | `LocalLockEnabled` | `bool?` | `true` | Per-provider | Acquire a local (in-process) lock before calling the value factory. | | `LocalLockTimeout` | `TimeSpan?` | `00:00:00.500` | Per-provider | Max wait to acquire the local lock before bypassing it. | | `DistributedLockEnabled` | `bool?` | `null` | Per-provider | Inert for this provider; present to satisfy `IMultilayerCacheOptions`. Startup validation still applies. | diff --git a/samples/UiPath.Caching.Sample/appsettings.all.json b/samples/UiPath.Caching.Sample/appsettings.all.json index ff83049..4d851fc 100644 --- a/samples/UiPath.Caching.Sample/appsettings.all.json +++ b/samples/UiPath.Caching.Sample/appsettings.all.json @@ -26,7 +26,7 @@ // LargeValueThreshold: bytes; writes above this are logged when AuditEnabled "LargeValueThreshold": 20000, // ConnectionMonitorEnabled: health-check Redis connection app-wide - "ConnectionMonitorEnabled": false, + "ConnectionMonitorEnabled": true, // KeyCasing: whether cache keys compare case-insensitively; the distributed cache is always sensitive "KeyCasing": "Insensitive", // LocalLockPoolSize: semaphore pool size for default local lock (allocation hint, not a hard cap) @@ -255,8 +255,10 @@ // CompactionPercentage: fraction of SizeLimit to free when the limit is hit; null = runtime default (0.05) "CompactionPercentage": null, // SizeProvider: set in code; see how-to/telemetry-and-strategies.md - // UseLocalOnlyWhenDisconnected: null = false; true = serve L1-only when L2 is disconnected + // UseLocalOnlyWhenDisconnected: null = true: serve and keep L1-only while L2 is disconnected; false = answer misses "UseLocalOnlyWhenDisconnected": null, + // ClearLocalOnReconnect: only with broadcast over RedisPubSub; null = true: clear L1 once it reconnects, since invalidations sent meanwhile were missed + "ClearLocalOnReconnect": null, // LocalMaxExpirationDisconnected: L1 TTL cap while L2 is disconnected; limits stale-read window "LocalMaxExpirationDisconnected": "0:00:30", // LocalLockEnabled: acquire a local (in-process) lock before calling the value factory @@ -303,9 +305,11 @@ // CompactionPercentage: fraction of SizeLimit to free when the limit is hit; null = runtime default (0.05) "CompactionPercentage": null, // SizeProvider: set in code; see how-to/telemetry-and-strategies.md - // UseLocalOnlyWhenDisconnected: inert for in-memory provider; present to satisfy IMultilayerCacheOptions contract + // UseLocalOnlyWhenDisconnected: only with broadcast over Redis; null = true: keep serving L1 while the broadcast connection is down "UseLocalOnlyWhenDisconnected": null, - // LocalMaxExpirationDisconnected: inert for in-memory provider; present to satisfy IMultilayerCacheOptions contract + // ClearLocalOnReconnect: only with broadcast over RedisPubSub; null = true: clear L1 once the broadcast connection recovers + "ClearLocalOnReconnect": null, + // LocalMaxExpirationDisconnected: only with broadcast over Redis: L1 lifetime cap for values written while the broadcast connection is down "LocalMaxExpirationDisconnected": "0:00:30", // LocalLockEnabled: acquire a local (in-process) lock before calling the value factory "LocalLockEnabled": true, diff --git a/src/UiPath.Caching.Abstractions/CacheOptions.cs b/src/UiPath.Caching.Abstractions/CacheOptions.cs index ff25023..84d8f08 100644 --- a/src/UiPath.Caching.Abstractions/CacheOptions.cs +++ b/src/UiPath.Caching.Abstractions/CacheOptions.cs @@ -37,7 +37,7 @@ public class CacheOptions public int LargeValueThreshold { get; set; } = 20_000; - public bool ConnectionMonitorEnabled { get; set; } + public bool ConnectionMonitorEnabled { get; set; } = true; /// diff --git a/src/UiPath.Caching.Abstractions/IConnectionState.cs b/src/UiPath.Caching.Abstractions/IConnectionState.cs index f0eabd6..a60a2ec 100644 --- a/src/UiPath.Caching.Abstractions/IConnectionState.cs +++ b/src/UiPath.Caching.Abstractions/IConnectionState.cs @@ -8,6 +8,6 @@ public interface IConnectionState event EventHandler? OnReconnected; - /// Non-blocking snapshot of the current connection state; returns false while connecting or after a failed connect, and never blocks or throws. + /// Non-blocking snapshot of the current connection state; never blocks or throws. A connector that connects on first use reports true until it has a connection that is down, so that first command can open it. bool IsConnected { get; } } diff --git a/src/UiPath.Caching/Broadcast/ChangeToken.cs b/src/UiPath.Caching/Broadcast/ChangeToken.cs index c11f300..eb4dbea 100644 --- a/src/UiPath.Caching/Broadcast/ChangeToken.cs +++ b/src/UiPath.Caching/Broadcast/ChangeToken.cs @@ -2,7 +2,7 @@ namespace UiPath.Caching.Broadcast; -public sealed partial class ChangeToken : ICacheChangeToken, IKeyedObserver, IDisposable +public sealed partial class ChangeToken : ICacheChangeToken, IKeyedObserver, IMissedEventsObserver, IDisposable { private readonly string _key; private readonly KeyMasker _masker; @@ -16,8 +16,21 @@ public sealed partial class ChangeToken : ICacheChangeToken, IKeyedObserver callback, object? state)> _callbacks = []; + // An event and a loss report can arrive on different threads; each notification and registration takes its turn. +#if NET9_0_OR_GREATER + private readonly Lock _notifyLock = new(); +#else + private readonly object _notifyLock = new(); +#endif + + // Set once a removal, a loss or an error expired the entry: a refresh the dispatcher delivers after it would re-insert the old value. + private bool _expired; + public ChangeToken( string key, ITopic topic, @@ -40,8 +53,10 @@ internal ChangeToken( ISet? acceptedEvents, KeyMasker masker, Type? entryType, - CacheKey callerKey) + CacheKey callerKey, + bool ignoreSubscriptionGaps = false) { + _ignoreSubscriptionGaps = ignoreSubscriptionGaps; _masker = masker; _entryType = entryType; _callerKey = callerKey; @@ -105,11 +120,34 @@ public void OnNext(ICacheEvent cacheEvent) } } + void IMissedEventsObserver.OnEventsMissed(MissedEventsReason reason) + { + if (reason == MissedEventsReason.SubscriptionGap && _ignoreSubscriptionGaps) + { + return; + } + + LogClearLocalCacheOnLoss(Logged(), _topic); + Notify(); + } + public IDisposable RegisterChangeCallback(Action callback, object? state) { - _callbacks.Add(new(callback, state)); + lock (_notifyLock) + { + // A token can change while it subscribes, before its holder registers: a late registration runs at once. + if (HasChanged) + { + callback(state); + } + else + { + _callbacks.Add(new(callback, state)); + } + } + return this; } @@ -124,14 +162,25 @@ public void Dispose() => private void Notify(CacheEventData? data = default) { - HasChanged = true; - if(data?.Properties != null) + lock (_notifyLock) { - ExtractExpiration(data.Properties); - ExtractMetadata(data.Properties); - } + HasChanged = true; + if (data?.Properties != null && !_expired) + { + ExtractExpiration(data.Properties); + ExtractMetadata(data.Properties); + } + else + { + // A removal or a loss carries nothing to refresh with: what an earlier refresh left would re-insert the value. + _expired = true; + Expiration = null; + Metadata = null; + MetadataHasChanged = false; + } - _callbacks.ForEach(kv => kv.callback(kv.state)); + _callbacks.ForEach(kv => kv.callback(kv.state)); + } } private bool IsAcceptedEvent(ICacheEvent cacheEvent) @@ -205,6 +254,9 @@ private void ExtractExpiration(IDictionary properties) [LoggerMessage(Level = LogLevel.Debug, Message = "Clear local cache {Key},{Topic}")] private partial void LogClearLocalCacheOnError(Exception error, LoggedKey key, TopicKey topic); + [LoggerMessage(Level = LogLevel.Debug, Message = "Clear local cache {Key}: topic {Topic} lost invalidations")] + private partial void LogClearLocalCacheOnLoss(LoggedKey key, TopicKey topic); + [LoggerMessage(Level = LogLevel.Debug, Message = "Clear local cache key {Key}. Topic:{Topic}, Id {EventId}, Source:{EventSource}")] private partial void LogClearLocalCacheKey(LoggedKey key, TopicKey topic, string? eventId, Uri? eventSource); diff --git a/src/UiPath.Caching/Broadcast/ChangeTokenFactory.cs b/src/UiPath.Caching/Broadcast/ChangeTokenFactory.cs index 72755ea..44e805a 100644 --- a/src/UiPath.Caching/Broadcast/ChangeTokenFactory.cs +++ b/src/UiPath.Caching/Broadcast/ChangeTokenFactory.cs @@ -36,10 +36,10 @@ public ICacheChangeToken Create(string token, ITopic topic, string CreateCore(token, topic, cacheName, entryType, _maskers.GetOrAdd(cacheName, name => KeyMasker.For(_keyMaskingPolicy, name)), callerKey: token); /// A tier built with its own policy, such as a private cache behind the distributed adapter, passes the masker it was given. - ICacheChangeToken IMaskedChangeTokenFactory.Create(string token, ITopic topic, string cacheName, Type entryType, KeyMasker masker, CacheKey callerKey) => - CreateCore(token, topic, cacheName, entryType, masker, callerKey); + ICacheChangeToken IMaskedChangeTokenFactory.Create(string token, ITopic topic, string cacheName, Type entryType, KeyMasker masker, CacheKey callerKey, bool ignoreSubscriptionGaps) => + CreateCore(token, topic, cacheName, entryType, masker, callerKey, ignoreSubscriptionGaps); - private ChangeToken CreateCore(string token, ITopic topic, string cacheName, Type entryType, KeyMasker masker, CacheKey callerKey) + private ChangeToken CreateCore(string token, ITopic topic, string cacheName, Type entryType, KeyMasker masker, CacheKey callerKey, bool ignoreSubscriptionGaps = false) { if (_logger.IsEnabled(LogLevel.Trace)) { @@ -47,7 +47,7 @@ private ChangeToken CreateCore(string token, ITopic topic, strin } var acceptedEvents = KnownCacheProviderNames.InMemory.Equals(cacheName, StringComparison.OrdinalIgnoreCase) ? MemoryAcceptedEvents : null; - return new ChangeToken(token, topic, _sourceUri, _serializer, _loggerFactory.CreateLogger>(), _telemetryProvider, acceptedEvents, masker, entryType, callerKey); + return new ChangeToken(token, topic, _sourceUri, _serializer, _loggerFactory.CreateLogger>(), _telemetryProvider, acceptedEvents, masker, entryType, callerKey, ignoreSubscriptionGaps); } [LoggerMessage(Level = LogLevel.Trace, Message = "Create change token. topic {TopicKey} token {Token} source {SourceUri}")] diff --git a/src/UiPath.Caching/Broadcast/ChannelHelper.cs b/src/UiPath.Caching/Broadcast/ChannelHelper.cs index 9e82661..43aa8e9 100644 --- a/src/UiPath.Caching/Broadcast/ChannelHelper.cs +++ b/src/UiPath.Caching/Broadcast/ChannelHelper.cs @@ -4,20 +4,28 @@ namespace UiPath.Caching.Broadcast; internal static class ChannelHelper { - public static Channel Create(bool unbounded, int capacity, BoundedChannelFullMode fullMode) => unbounded - ? Channel.CreateUnbounded(new UnboundedChannelOptions + /// A bounded channel reports each item a full channel drops to . + public static Channel Create(bool unbounded, int capacity, BoundedChannelFullMode fullMode, Action? itemDropped = null) + { + if (unbounded) + { + return Channel.CreateUnbounded(new UnboundedChannelOptions { SingleReader = true, SingleWriter = true, AllowSynchronousContinuations = false, - }) - : Channel.CreateBounded(new BoundedChannelOptions(capacity) - { - FullMode = fullMode, - SingleReader = true, - SingleWriter = true, - AllowSynchronousContinuations = false, }); + } + + var options = new BoundedChannelOptions(capacity) + { + FullMode = fullMode, + SingleReader = true, + SingleWriter = true, + AllowSynchronousContinuations = false, + }; + return Channel.CreateBounded(options, itemDropped); + } public static int CalculateBoundedCapacity(int consumerCapacity, int pollBatchSize) => consumerCapacity > 0 diff --git a/src/UiPath.Caching/Broadcast/DroppedEvents.cs b/src/UiPath.Caching/Broadcast/DroppedEvents.cs new file mode 100644 index 0000000..0a873fd --- /dev/null +++ b/src/UiPath.Caching/Broadcast/DroppedEvents.cs @@ -0,0 +1,58 @@ +namespace UiPath.Caching.Broadcast; + +/// Turns events dropped at a full dispatcher channel into one expiry at a time, off the thread that dropped them. +internal sealed partial class DroppedEvents(TopicKey topicKey, Action expire, ILogger logger) +{ + private const int Idle = 0; + private const int Running = 1; + private const int Rerun = 2; + + private readonly ILogger _logger = logger; + private int _state; + + public void Dropped() + { + while (true) + { + switch (Volatile.Read(ref _state)) + { + case Idle when Interlocked.CompareExchange(ref _state, Running, Idle) == Idle: + ThreadPool.UnsafeQueueUserWorkItem(static self => self.Run(), this, preferLocal: false); + return; + case Running when Interlocked.CompareExchange(ref _state, Rerun, Running) == Running: + case Rerun: + return; + } + } + } + + private void Run() + { + while (true) + { + LogEventsDropped(topicKey); + try + { + expire(); + } + catch (Exception ex) + { + LogExpireFailed(ex, topicKey); + } + + if (Interlocked.CompareExchange(ref _state, Idle, Running) == Running) + { + return; + } + + // A drop came in during that expiry: run once more, on this worker. + Volatile.Write(ref _state, Running); + } + } + + [LoggerMessage(Level = LogLevel.Warning, Message = "Topic {TopicKey} dropped events at its full dispatcher channel; expiring what it keeps")] + private partial void LogEventsDropped(TopicKey topicKey); + + [LoggerMessage(Level = LogLevel.Error, Message = "Expiring topic {TopicKey} after dropped events failed")] + private partial void LogExpireFailed(Exception ex, TopicKey topicKey); +} diff --git a/src/UiPath.Caching/Broadcast/GapFilteringTopic.cs b/src/UiPath.Caching/Broadcast/GapFilteringTopic.cs new file mode 100644 index 0000000..f31cd07 --- /dev/null +++ b/src/UiPath.Caching/Broadcast/GapFilteringTopic.cs @@ -0,0 +1,52 @@ +namespace UiPath.Caching.Broadcast; + +/// +/// The topic a custom change token factory is handed for a cache that keeps its entries over a subscription gap: its +/// observers hear a known loss but not a gap, which that factory has no other way to learn the cache chose. +/// +internal sealed class GapFilteringTopic(ITopic inner) : ITopic +{ + public TopicKey TopicKey => inner.TopicKey; + + public EventHandler? OnDisposed + { + get => inner.OnDisposed; + set => inner.OnDisposed = value; + } + + public IDisposable Subscribe(IObserver observer) => inner.Subscribe(observer switch + { + IKeyedObserver and IMissedEventsObserver => new KeyedFilter(observer), + IMissedEventsObserver => new Filter(observer), + _ => observer, + }); + + public ValueTask PublishAsync(ICacheEvent @event, CancellationToken token = default) => inner.PublishAsync(@event, token); + + // The topic belongs to its provider, not to this view of it. + public void Dispose() + { + } + + private class Filter(IObserver observer) : IObserver, IMissedEventsObserver + { + public void OnNext(ICacheEvent value) => observer.OnNext(value); + + public void OnError(Exception error) => observer.OnError(error); + + public void OnCompleted() => observer.OnCompleted(); + + public void OnEventsMissed(MissedEventsReason reason) + { + if (reason != MissedEventsReason.SubscriptionGap) + { + ((IMissedEventsObserver)observer).OnEventsMissed(reason); + } + } + } + + private sealed class KeyedFilter(IObserver observer) : Filter(observer), IKeyedObserver + { + public string Key { get; } = ((IKeyedObserver)observer).Key; + } +} diff --git a/src/UiPath.Caching/Broadcast/IEventSubject.cs b/src/UiPath.Caching/Broadcast/IEventSubject.cs index 8ab92f2..c10ee0a 100644 --- a/src/UiPath.Caching/Broadcast/IEventSubject.cs +++ b/src/UiPath.Caching/Broadcast/IEventSubject.cs @@ -7,4 +7,7 @@ public interface IEventSubject : IDisposable where T : IEvent void OnNext(T value); void OnCompleted(); + + /// Expire what the observers keep, without ending their subscriptions: events meant for them were lost. + void Invalidate(MissedEventsReason reason); } diff --git a/src/UiPath.Caching/Broadcast/IMaskedChangeTokenFactory.cs b/src/UiPath.Caching/Broadcast/IMaskedChangeTokenFactory.cs index 2b02090..3a0f66f 100644 --- a/src/UiPath.Caching/Broadcast/IMaskedChangeTokenFactory.cs +++ b/src/UiPath.Caching/Broadcast/IMaskedChangeTokenFactory.cs @@ -3,5 +3,5 @@ namespace UiPath.Caching.Broadcast; /// Lets a tier that was built with its own masking policy pass that decision into the change tokens it creates. internal interface IMaskedChangeTokenFactory { - ICacheChangeToken Create(string token, ITopic topic, string cacheName, Type entryType, KeyMasker masker, CacheKey callerKey); + ICacheChangeToken Create(string token, ITopic topic, string cacheName, Type entryType, KeyMasker masker, CacheKey callerKey, bool ignoreSubscriptionGaps = false); } diff --git a/src/UiPath.Caching/Broadcast/IMissedEventsObserver.cs b/src/UiPath.Caching/Broadcast/IMissedEventsObserver.cs new file mode 100644 index 0000000..f4e4839 --- /dev/null +++ b/src/UiPath.Caching/Broadcast/IMissedEventsObserver.cs @@ -0,0 +1,11 @@ +namespace UiPath.Caching.Broadcast; + +/// +/// An observer a subject can expire, without ending its subscription, when events meant for it were lost. A subject may +/// call it while an runs on another thread, and more than once for one loss when the +/// observer joins as it is reported, so expiring must be idempotent. +/// +public interface IMissedEventsObserver +{ + void OnEventsMissed(MissedEventsReason reason); +} diff --git a/src/UiPath.Caching/Broadcast/KeyedSubject.cs b/src/UiPath.Caching/Broadcast/KeyedSubject.cs index f92fa2a..8b6401c 100644 --- a/src/UiPath.Caching/Broadcast/KeyedSubject.cs +++ b/src/UiPath.Caching/Broadcast/KeyedSubject.cs @@ -8,6 +8,7 @@ internal sealed partial class KeyedSubject : IEventSubject where T : IEven private readonly ConcurrentDictionary, byte>> _keyedObservers = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary, byte> _broadcastObservers = new(); private readonly object _keyedLock = new(); + private readonly ILogger _logger; private readonly TimeSpan _slowObserverThreshold; private volatile bool _completed; @@ -26,18 +27,21 @@ public IDisposable Subscribe(IObserver observer) return Disposable.Empty; } + // Completion may run while this registers: checked again after, and whoever removes the observer completes it, once. if (observer is IKeyedObserver keyed) { + ConcurrentDictionary, byte> inner; lock (_keyedLock) { - var inner = _keyedObservers.GetOrAdd(keyed.Key, _ => new ConcurrentDictionary, byte>()); + inner = _keyedObservers.GetOrAdd(keyed.Key, _ => new ConcurrentDictionary, byte>()); inner.TryAdd(observer, 0); } - return new Subscription(this, keyed.Key, observer); + + return CompletedMeanwhile(inner, observer) ? Disposable.Empty : new Subscription(this, keyed.Key, observer); } _broadcastObservers.TryAdd(observer, 0); - return new Subscription(this, null, observer); + return CompletedMeanwhile(_broadcastObservers, observer) ? Disposable.Empty : new Subscription(this, null, observer); } public void OnNext(T value) @@ -70,21 +74,61 @@ public void OnCompleted() { foreach (var kvp in inner) { - SafeOnCompleted(kvp.Key); + CompleteIfRemoved(inner, kvp.Key); } } foreach (var kvp in _broadcastObservers) { - SafeOnCompleted(kvp.Key); + CompleteIfRemoved(_broadcastObservers, kvp.Key); } _keyedObservers.Clear(); - _broadcastObservers.Clear(); } public void Dispose() => OnCompleted(); + /// Expires every observer and keeps it subscribed. + public void Invalidate(MissedEventsReason reason) + { + if (_completed) + { + return; + } + + foreach (var inner in _keyedObservers.Values) + { + foreach (var kvp in inner) + { + SafeInvalidate(kvp.Key, reason); + } + } + + foreach (var kvp in _broadcastObservers) + { + SafeInvalidate(kvp.Key, reason); + } + } + + private bool CompletedMeanwhile(ConcurrentDictionary, byte> observers, IObserver observer) + { + if (!_completed) + { + return false; + } + + CompleteIfRemoved(observers, observer); + return true; + } + + private void CompleteIfRemoved(ConcurrentDictionary, byte> observers, IObserver observer) + { + if (observers.TryRemove(observer, out _)) + { + SafeOnCompleted(observer); + } + } + private void SafeOnNext(IObserver observer, T value) { var start = Stopwatch.GetTimestamp(); @@ -103,6 +147,23 @@ private void SafeOnNext(IObserver observer, T value) } } + private void SafeInvalidate(IObserver observer, MissedEventsReason reason) + { + if (observer is not IMissedEventsObserver missed) + { + return; + } + + try + { + missed.OnEventsMissed(reason); + } + catch (Exception ex) + { + LogObserverInvalidateFailed(ex); + } + } + private void SafeOnCompleted(IObserver observer) { try @@ -143,6 +204,9 @@ private void Unsubscribe(string? key, IObserver observer) [LoggerMessage(Level = LogLevel.Warning, Message = "Observer threw in OnCompleted; continuing.")] private partial void LogObserverOnCompletedFailed(Exception ex); + [LoggerMessage(Level = LogLevel.Warning, Message = "Observer threw in OnEventsMissed; continuing.")] + private partial void LogObserverInvalidateFailed(Exception ex); + [LoggerMessage(Level = LogLevel.Warning, Message = "Slow observer {Observer} took {ElapsedMs} ms in OnNext for event {EventId}.")] private partial void LogObserverSlow(string? observer, double elapsedMs, string? eventId); diff --git a/src/UiPath.Caching/Broadcast/MissedEventsReason.cs b/src/UiPath.Caching/Broadcast/MissedEventsReason.cs new file mode 100644 index 0000000..b6b62fb --- /dev/null +++ b/src/UiPath.Caching/Broadcast/MissedEventsReason.cs @@ -0,0 +1,11 @@ +namespace UiPath.Caching.Broadcast; + +/// Why a subject expires what its observers keep. +public enum MissedEventsReason +{ + /// Events are known lost, such as stream entries trimmed before this node read them. + Lost, + + /// The subscription was not in place for a while, so events published then may not have arrived. + SubscriptionGap, +} diff --git a/src/UiPath.Caching/Broadcast/Redis/ConsumerGroups.cs b/src/UiPath.Caching/Broadcast/Redis/ConsumerGroups.cs new file mode 100644 index 0000000..05d3fc5 --- /dev/null +++ b/src/UiPath.Caching/Broadcast/Redis/ConsumerGroups.cs @@ -0,0 +1,80 @@ +namespace UiPath.Caching.Broadcast.Redis; + +/// +/// Creates a group with the count of entries before where it starts, which Redis 7.0 otherwise leaves unknown until the +/// group reads, so a trim before its first read shows in the counts. +/// +internal static class ConsumerGroups +{ + public static bool Create(IDatabase database, RedisKey stream, RedisValue group, RedisValue position) + { + if (position == StreamPosition.NewMessages && Start(() => database.StreamInfo(stream)) is { } start) + { + try + { + database.Execute("XGROUP", Arguments(stream, group, start)); + return true; + } + catch (RedisServerException ex) when (ex.Message != StreamConstants.ConsumerGroupNameExistsErrorMessage) + { + // Created without a count below. + } + } + + return database.StreamCreateConsumerGroup(stream, group, position); + } + + public static async Task CreateAsync(IDatabase database, RedisKey stream, RedisValue group, RedisValue position) + { + if (position == StreamPosition.NewMessages && await StartAsync(database, stream).ConfigureAwait(false) is { } start) + { + try + { + await database.ExecuteAsync("XGROUP", Arguments(stream, group, start)).ConfigureAwait(false); + return true; + } + catch (RedisServerException ex) when (ex.Message != StreamConstants.ConsumerGroupNameExistsErrorMessage) + { + // Created without a count below. + } + } + + return await database.StreamCreateConsumerGroupAsync(stream, group, position).ConfigureAwait(false); + } + + private static async Task StartAsync(IDatabase database, RedisKey stream) + { + try + { + return Start(await database.StreamInfoAsync(stream).ConfigureAwait(false)); + } + catch (RedisServerException ex) + { + return Missing(ex); + } + } + + private static ReadCount? Start(Func info) + { + try + { + return Start(info()); + } + catch (RedisServerException ex) + { + return Missing(ex); + } + } + + // Starting at the newest entry rather than $ keeps the count exact if an entry is added before the group exists. + // Below Redis 7.0 there is no max-deleted-entry-id, nor ENTRIESREAD. + private static ReadCount? Start(StreamInfo info) => + info.MaxDeletedEntryId.IsNull ? null : new ReadCount(info.LastGeneratedId, info.EntriesAdded); + + // Nothing comes before 0-0, whoever creates the stream first; XINFO denied leaves the count unknown. + private static ReadCount? Missing(RedisServerException ex) => + ex.Message.StartsWith("ERR no such key", StringComparison.OrdinalIgnoreCase) ? new ReadCount("0-0", 0) : null; + + private static object[] Arguments(RedisKey stream, RedisValue group, ReadCount start) => + ["CREATE", stream, group, start.LastDelivered, "MKSTREAM", "ENTRIESREAD", start.EntriesRead]; +} diff --git a/src/UiPath.Caching/Broadcast/Redis/ReadCount.cs b/src/UiPath.Caching/Broadcast/Redis/ReadCount.cs new file mode 100644 index 0000000..7a7e396 --- /dev/null +++ b/src/UiPath.Caching/Broadcast/Redis/ReadCount.cs @@ -0,0 +1,4 @@ +namespace UiPath.Caching.Broadcast.Redis; + +/// How many entries a group had read when its last delivery was . +internal readonly record struct ReadCount(RedisValue LastDelivered, long EntriesRead); diff --git a/src/UiPath.Caching/Broadcast/Redis/RedisPubSubSubjectWriter.cs b/src/UiPath.Caching/Broadcast/Redis/RedisPubSubSubjectWriter.cs index 97a7609..fb7a284 100644 --- a/src/UiPath.Caching/Broadcast/Redis/RedisPubSubSubjectWriter.cs +++ b/src/UiPath.Caching/Broadcast/Redis/RedisPubSubSubjectWriter.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Threading.Channels; namespace UiPath.Caching.Broadcast.Redis; @@ -15,10 +16,25 @@ internal sealed partial class RedisPubSubSubjectWriter : IDisposable private readonly TimeSpan _timerPeriod; private readonly TimeSpan _timerDueTime; private readonly Timer _subscribeTimer; + private readonly Action _onSubscriptionGap; + private readonly Action _onDropped; + private readonly object _gate = new(); + + // Under _gate: the subscription connections down now. + private readonly HashSet _subscriptionsDown = []; private bool _disposed; private Action? _unsubscribe; private int _subscribing; + // Under _gate: each reconnect starts a generation, and only a subscription made in the current one stops the timer. + private int _generation; + private int _activeGeneration = -1; + private bool _gap; + + // Under _gate: whether a source that names no connection reported a failure. + private bool _unnamedFailure; + private int _activations; + public RedisPubSubSubjectWriter( Uri sourceUri, RedisChannel redisChannel, @@ -26,8 +42,12 @@ public RedisPubSubSubjectWriter( ChannelWriter channelWriter, IEventFormatterProxy formatter, RedisPubSubTopicOptions options, - ILogger logger) + ILogger logger, + Action onSubscriptionGap, + Action onDropped) { + _onSubscriptionGap = onSubscriptionGap; + _onDropped = onDropped; _redis = redis; _channelWriter = channelWriter; _formatter = formatter; @@ -35,6 +55,8 @@ public RedisPubSubSubjectWriter( _sourceUri = sourceUri; _redisChannel = redisChannel; _redis.OnReconnected += OnReconnected; + _redis.OnConnectionFailed += OnConnectionFailed; + _redis.OnConnectionRestored += OnConnectionRestored; _handler = (_, value) => OnMessage(value); _timerPeriod = options.SubscriberTimeout > TimeSpan.Zero ? options.SubscriberTimeout.Value : TimeSpan.FromMilliseconds(_redis.Subscriber.Multiplexer.TimeoutMilliseconds); _timerDueTime = options.SubscriberDueTime == null ? _timerPeriod.Multiply(0.5) : options.SubscriberDueTime.Value; @@ -46,12 +68,34 @@ public void Dispose() if (!_disposed) { _redis.OnReconnected -= OnReconnected; + _redis.OnConnectionFailed -= OnConnectionFailed; + _redis.OnConnectionRestored -= OnConnectionRestored; _subscribeTimer.Dispose(); Unsubscribe(); } _disposed = true; } + /// The topic is gaining an observer, whose entry a publication before the subscription is in place could leave stale. + /// A mark for . + internal int MarkSubscribed() + { + lock (_gate) + { + _gap |= _activeGeneration != _generation; + return _activations; + } + } + + /// Whether a subscription went in place since , expiring the observers already there. + internal bool ActivatedSince(int mark) + { + lock (_gate) + { + return _activations != mark; + } + } + private void Subscribe(object? state) { if (Interlocked.CompareExchange(ref _subscribing, 1, 0) != 0) @@ -60,6 +104,19 @@ private void Subscribe(object? state) } LogSubscribeChannel(_redisChannel); + int generation; + lock (_gate) + { + // A tick already queued when the timer stopped finds this generation in place and leaves it. + if (_activeGeneration == _generation) + { + Interlocked.Exchange(ref _subscribing, 0); + return; + } + + generation = _generation; + } + try { _unsubscribe?.Invoke(); @@ -67,7 +124,27 @@ private void Subscribe(object? state) var subscriber = _redis.Subscriber; subscriber.Subscribe(_redisChannel, _handler); _unsubscribe = () => subscriber.Unsubscribe(_redisChannel, _handler, CommandFlags.FireAndForget); - _subscribeTimer.Change(Timeout.Infinite, Timeout.Infinite); + bool gap; + lock (_gate) + { + // A reconnect during this attempt left the timer armed, so it subscribes again on the new connection. + if (generation != _generation) + { + return; + } + + _subscribeTimer.Change(Timeout.Infinite, Timeout.Infinite); + _activeGeneration = generation; + _activations++; + gap = _gap; + _gap = false; + } + + // Entries cached before this subscription was in place may have missed what was published meanwhile. + if (gap) + { + _onSubscriptionGap(); + } } catch (Exception ex) { @@ -86,7 +163,15 @@ private void OnReconnected(object? sender, EventArgs e) return; } - _subscribeTimer.Change(_timerDueTime, _timerPeriod); + lock (_gate) + { + // The retired connection will not report its restores; the resubscribe this arms covers what it dropped. + _subscriptionsDown.Clear(); + _unnamedFailure = false; + _generation++; + _gap = true; + _subscribeTimer.Change(_timerDueTime, _timerPeriod); + } } private void OnMessage(RedisValue value) @@ -110,7 +195,7 @@ private void OnMessage(RedisValue value) } else { - _channelWriter.TryWrite(ev); + Write(ev); } } else @@ -125,6 +210,65 @@ private void OnMessage(RedisValue value) } } + private void Write(T ev) + { + // A full channel in Wait mode refuses the write; the event, and the invalidation it carried, is gone. + if (!_channelWriter.TryWrite(ev)) + { + _onDropped(); + } + } + + private void OnConnectionFailed(object? sender, EventArgs e) + { + lock (_gate) + { + // Only the subscription connection carries publications; a failed command connection loses none of them. + if (e is ConnectionFailedEventArgs failed) + { + if (failed.ConnectionType == ConnectionType.Subscription) + { + _subscriptionsDown.Add(failed.EndPoint); + } + } + else + { + _unnamedFailure = true; + } + } + } + + private void OnConnectionRestored(object? sender, EventArgs e) + { + bool lost = false; + lock (_gate) + { + if (e is ConnectionFailedEventArgs restored) + { + // Expired once the last subscription connection that went down is back, not on an unrelated restore. + if (restored.ConnectionType == ConnectionType.Subscription + && (_subscriptionsDown.Remove(restored.EndPoint) || _unnamedFailure) + && _subscriptionsDown.Count == 0) + { + lost = true; + _unnamedFailure = false; + } + } + else if (_unnamedFailure || _subscriptionsDown.Count > 0) + { + lost = true; + _unnamedFailure = false; + _subscriptionsDown.Clear(); + } + } + + // Publications while the connection was down never arrived. Expired off the connection's own event thread. + if (lost && !_disposed) + { + ThreadPool.UnsafeQueueUserWorkItem(static self => self._onSubscriptionGap(), this, preferLocal: false); + } + } + private void Unsubscribe() { LogUnsubscribeChannel(_redisChannel); diff --git a/src/UiPath.Caching/Broadcast/Redis/RedisPubSubTopic.cs b/src/UiPath.Caching/Broadcast/Redis/RedisPubSubTopic.cs index 4b7341b..a6d2865 100644 --- a/src/UiPath.Caching/Broadcast/Redis/RedisPubSubTopic.cs +++ b/src/UiPath.Caching/Broadcast/Redis/RedisPubSubTopic.cs @@ -41,8 +41,9 @@ public RedisPubSubTopic( _logger = logger; _options = options; _subject = subjectFactory(); - var channel = ChannelHelper.Create(options.ConsumerCapacity < 1, options.ConsumerCapacity, options.FullMode); - _subscriber = new RedisPubSubSubjectWriter(sourceUri, _redisChannel, _redis, channel, _formatter, options, _logger); + var dropped = new DroppedEvents(topicKey, () => _subject.Invalidate(MissedEventsReason.Lost), _logger); + var channel = ChannelHelper.Create(options.ConsumerCapacity < 1, options.ConsumerCapacity, options.FullMode, _ => dropped.Dropped()); + _subscriber = new RedisPubSubSubjectWriter(sourceUri, _redisChannel, _redis, channel, _formatter, options, _logger, () => _subject.Invalidate(MissedEventsReason.SubscriptionGap), dropped.Dropped); _dispatcher = new EventDispatcher(topicKey, channel, _subject, _logger, _stopTokenSource.Token); } @@ -50,8 +51,20 @@ public RedisPubSubTopic( public EventHandler? OnDisposed { get; set; } - public IDisposable Subscribe(IObserver observer) => - _subject.Subscribe(observer); + public IDisposable Subscribe(IObserver observer) + { + // Publications are missed while the Redis subscription is not in place, so the mark makes the next one that goes in + // place expire this observer too. One that went in place while the observer was being added expired the others + // before reaching it, so this observer is told here. + var mark = _subscriber.MarkSubscribed(); + var subscription = _subject.Subscribe(observer); + if (_subscriber.ActivatedSince(mark) && observer is IMissedEventsObserver missed) + { + TellMissed(missed, MissedEventsReason.SubscriptionGap); + } + + return subscription; + } public async ValueTask PublishAsync(T @event, CancellationToken token = default) { @@ -100,9 +113,25 @@ public void Dispose() internal RedisPubSubTopicOptions GetResolvedOptionsForTests() => _options; + /// Tells an observer that joined as the subscription went in place that it may hold stale entries; what it throws is logged, not passed to Subscribe. + private void TellMissed(IMissedEventsObserver missed, MissedEventsReason reason) + { + try + { + missed.OnEventsMissed(reason); + } + catch (Exception ex) + { + LogObserverMissedEventsFailed(ex, TopicKey); + } + } + [LoggerMessage(Level = LogLevel.Trace, Message = "Publishing to topic {TopicKey} event {EventId}")] private partial void LogPublishing(TopicKey topicKey, string? eventId); + [LoggerMessage(Level = LogLevel.Warning, Message = "Observer threw in OnEventsMissed on topic {TopicKey}; it stays subscribed.")] + private partial void LogObserverMissedEventsFailed(Exception ex, TopicKey topicKey); + [LoggerMessage(Level = LogLevel.Warning, Message = "Error when publishing to Topic {TopicKey}")] private partial void LogPublishError(Exception ex, TopicKey topicKey); } diff --git a/src/UiPath.Caching/Broadcast/Redis/RedisStreamSubjectWriter.cs b/src/UiPath.Caching/Broadcast/Redis/RedisStreamSubjectWriter.cs index 9e5803d..a4238de 100644 --- a/src/UiPath.Caching/Broadcast/Redis/RedisStreamSubjectWriter.cs +++ b/src/UiPath.Caching/Broadcast/Redis/RedisStreamSubjectWriter.cs @@ -6,6 +6,32 @@ namespace UiPath.Caching.Broadcast.Redis; internal sealed partial class RedisStreamSubjectWriter : IDisposable where T : IEvent { + // Runs on every read, so it returns only what the check needs: XINFO STREAM carries the first and last entries whole. + private const string CheckAndReadScript = """ + local info = redis.call('XINFO', 'STREAM', KEYS[1]) + local stream = {} + for i = 1, #info, 2 do + local field = info[i] + if field == 'length' or field == 'last-generated-id' or field == 'max-deleted-entry-id' or field == 'entries-added' then + stream[#stream + 1] = field + stream[#stream + 1] = info[i + 1] + elseif field == 'first-entry' and info[i + 1] then + stream[#stream + 1] = field + stream[#stream + 1] = { info[i + 1][1] } + end + end + local groups = {} + for _, group in ipairs(redis.call('XINFO', 'GROUPS', KEYS[1])) do + for i = 1, #group, 2 do + if group[i] == 'name' and group[i + 1] == ARGV[1] then + groups[1] = group + end + end + end + local entries = redis.call('XREADGROUP', 'GROUP', ARGV[1], ARGV[2], 'COUNT', ARGV[3], 'STREAMS', KEYS[1], '>') + return { stream, groups, entries } + """; + private const string EventInvalid = "Caching." + nameof(RedisStreamSubjectWriter) + "." + nameof(DispatchEventsAsync) + ".InvalidEvent"; private const string EventReceived = "Caching." + nameof(RedisStreamSubjectWriter) + "." + nameof(DispatchEventsAsync) + ".EventReceived"; private const string PropTopicKey = "TopicKey"; @@ -23,11 +49,22 @@ internal sealed partial class RedisStreamSubjectWriter : IDisposable private readonly CancellationToken _cancelationToken; private readonly IFetchWaiter _waiter; private readonly SemaphoreSlim _retryGate = new(0, 1); + private readonly Action _onMessagesMissed; private bool _disposed; private RedisValue _lastId = StreamPosition.NewMessages; private int _consecutiveFailures; private volatile bool _unsupportedCommand; + private volatile bool _checkForGap; + // Set once the topic has subscribers or the group has been read: only then can a loss leave a local entry stale. + private volatile bool _inUse; + private bool _noScripts; + private bool _gapCheckDenied; + + // An empty read cannot move the group past a trim, so every later check finds the gap this one reported. + private StreamGap? _reportedGap; + private ReadCount? _readCount; + private int _losses; public RedisStreamSubjectWriter( RedisStreamContext context, @@ -39,6 +76,7 @@ public RedisStreamSubjectWriter( ICachingTelemetryProvider cachingTelemetryProvider, IRedisProfiler redisProfiler, IFetchWaiter waiter, + Action onMessagesMissed, CancellationToken stopToken) { _context = context; @@ -50,6 +88,7 @@ public RedisStreamSubjectWriter( _cachingTelemetryProvider = cachingTelemetryProvider; _redisProfiler = redisProfiler; _waiter = waiter; + _onMessagesMissed = onMessagesMissed; _stopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stopToken); _cancelationToken = _stopTokenSource.Token; _connectionState.OnReconnected += OnConnectionRecovered; @@ -77,12 +116,57 @@ public void Dispose() _writer.TryComplete(); } + /// The topic is gaining a subscriber, whose entry a loss can leave stale before this group is ever read. + /// A mark for . + internal int MarkSubscribed() + { + // Read before publishing _inUse: a loss before it cannot reach this subscriber, and one after it moves the mark. + var mark = Volatile.Read(ref _losses); + _inUse = true; + return mark; + } + + /// Whether a loss was reported since , expiring the subscribers already there. + internal bool LostSince(int mark) => Volatile.Read(ref _losses) != mark; + + /// Expires what the topic keeps for a loss that was not a trim, such as an event dropped at a full channel. + internal void ReportLoss() + { + // Counted first, so a subscriber joining while the others are told sees it and is told itself. + Interlocked.Increment(ref _losses); + _onMessagesMissed(); + } + + private static bool IsDenied(Exception ex) => + ex is RedisServerException && ex.Message.StartsWith("NOPERM", StringComparison.OrdinalIgnoreCase); + private static bool IsUnsupportedCommand(Exception ex) => ex is RedisCommandException || (ex is RedisServerException && ex.Message.Contains(StreamConstants.UnknownCommandErrorMessage, StringComparison.OrdinalIgnoreCase)); - private void OnConnectionRecovered(object? sender, EventArgs e) => ReleaseRetryGate(); + private void ReportMessagesMissed() + { + LogMessagesMissed(_context.Topic, _context.ConsumerGroup); + ReportLoss(); + } + + private void ReportGap(StreamGap? gap) + { + if (gap is not { } found || _reportedGap == found) + { + return; + } + + _reportedGap = found; + ReportMessagesMissed(); + } + + private void OnConnectionRecovered(object? sender, EventArgs e) + { + _checkForGap = true; + ReleaseRetryGate(); + } private void ReleaseRetryGate() { @@ -145,18 +229,37 @@ private async Task FetchBatch() { if (!_connectionState.IsConnected) { + // IsConnected turns true before the restored event fires, so the next read may come first. + _checkForGap = true; await _waiter.WaitAsync(_cancelationToken).ConfigureAwait(false); return; } - StreamEntry[] events = await _redis.Database.StreamReadGroupAsync( - _context.Topic, - _context.ConsumerGroup, - _context.ConsumerName, - StreamConstants.UndeliveredMessages, - _context.PollBatchSize).ConfigureAwait(false); + // A trim can reach unread entries between any two polls, so every read the group's position matters for is checked. + StreamEntry[] events; + if (_inUse && !_noScripts) + { + events = await CheckAndReadAsync().ConfigureAwait(false); + } + else + { + // Without XINFO a loss cannot be ruled out, so only a reconnect reports one rather than every read. + if (_inUse && (_checkForGap || !_gapCheckDenied)) + { + // A recovery raised during a check needs its own before the read moves the group past the evidence. + do + { + await CheckForGapAsync().ConfigureAwait(false); + } + while (_checkForGap && _inUse); + } + + events = await ReadAsync().ConfigureAwait(false); + } _consecutiveFailures = 0; + _inUse = true; + if (_unsupportedCommand) { _unsupportedCommand = false; @@ -193,16 +296,26 @@ private async Task ProcessException(Exception ex) return true; } - if (ex.Message.StartsWith("NOGROUP", StringComparison.OrdinalIgnoreCase)) + if (ex.Message.Contains("NOGROUP", StringComparison.OrdinalIgnoreCase)) { if(id != StreamPosition.NewMessages) { LogRecreatingTopic(_context.Topic, _context.ConsumerGroup, id); } + if (_inUse) + { + // The group or the whole stream was removed while in use, so what it held past the last read is gone. + _inUse = false; + _checkForGap = false; + _reportedGap = null; + _readCount = null; + ReportMessagesMissed(); + } + try { - await _redis.Database.StreamCreateConsumerGroupAsync(_context.Topic, _context.ConsumerGroup, id).ConfigureAwait(false); + await ConsumerGroups.CreateAsync(_redis.Database, _context.Topic, _context.ConsumerGroup, id).ConfigureAwait(false); } catch (RedisServerException rex) when (rex.Message == StreamConstants.ConsumerGroupNameExistsErrorMessage) { @@ -216,7 +329,8 @@ private async Task ProcessException(Exception ex) } else { - LogFetchLoopError(ex); + _checkForGap = true; + LogFetchLoopError(ex); } await BackoffAsync().ConfigureAwait(false); @@ -250,6 +364,78 @@ private async Task WaitForRetryAsync(TimeSpan delay) } } + private Task ReadAsync() => _redis.Database.StreamReadGroupAsync( + _context.Topic, + _context.ConsumerGroup, + _context.ConsumerName, + StreamConstants.UndeliveredMessages, + _context.PollBatchSize); + + /// Checks for a gap and reads in one script, so a trim cannot land between the check and the read. + private async Task CheckAndReadAsync() + { + _checkForGap = false; + RedisResult reply; + try + { + reply = await _redis.Database.ScriptEvaluateAsync( + CheckAndReadScript, + [_context.Topic], + [_context.ConsumerGroup, _context.ConsumerName, _context.PollBatchSize]).ConfigureAwait(false); + } + catch (RedisServerException ex) when (ex.Message.Contains("no such key", StringComparison.OrdinalIgnoreCase)) + { + // A missing stream fails the read with NOGROUP, which reports the loss. + return await ReadAsync().ConfigureAwait(false); + } + catch (Exception ex) when (IsUnsupportedCommand(ex) || IsDenied(ex)) + { + // Scripts are off on this server: check and read apart from now on. + LogScriptsUnavailable(ex, _context.Topic); + _noScripts = true; + _checkForGap = true; + return []; + } + + var parts = StreamIds.Items(reply); + ReportGap(StreamIds.Gap(parts[0], parts[1], _context.ConsumerGroup, ref _readCount)); + + return StreamIds.Entries(parts[2]); + } + + /// Entries trimmed before the group read them were never delivered. + private async Task CheckForGapAsync() + { + // Cleared first, so a recovery raised while the check awaits arms the next one. + _checkForGap = false; + StreamInfo stream; + StreamGroupInfo[] groups; + try + { + stream = await _redis.Database.StreamInfoAsync(_context.Topic).ConfigureAwait(false); + groups = await _redis.Database.StreamGroupInfoAsync(_context.Topic).ConfigureAwait(false); + } + catch (RedisServerException ex) when (ex.Message.StartsWith("ERR no such key", StringComparison.OrdinalIgnoreCase)) + { + // A missing stream fails the read with NOGROUP, which reports the loss. + return; + } + catch (Exception ex) when (IsUnsupportedCommand(ex) || IsDenied(ex)) + { + // XINFO is denied or renamed away, so a loss cannot be ruled out: report one rather than retry a check that cannot succeed. + LogGapCheckDenied(ex, _context.Topic); + _gapCheckDenied = true; + ReportMessagesMissed(); + return; + } + + var group = Array.Find(groups, g => g.Name == _context.ConsumerGroup); + if (group.Name is not null) + { + ReportGap(StreamIds.Gap(stream, group, ref _readCount)); + } + } + private async Task DispatchEventsAsync(StreamEntry[] events) { List ids = new(events.Length); @@ -315,7 +501,11 @@ private async ValueTask DispatchValidEventAsync(T ev, StreamEntry @event, List(streamOptions.ConsumerCapacity < 0, capacity, streamOptions.FullMode); - _subscriber = new RedisStreamSubjectWriter(_context, _connectionState, _redis, channel, _formatter, _logger, _cachingTelemetryProvider, redisProfiler, _waiter, _stopTokenSource.Token); + // Run later on the pool, by which time the writer the drop reports through is in place. + var dropped = new DroppedEvents(topicKey, () => _subscriber!.ReportLoss(), _logger); + var channel = ChannelHelper.Create(streamOptions.ConsumerCapacity < 0, capacity, streamOptions.FullMode, _ => dropped.Dropped()); + _subscriber = new RedisStreamSubjectWriter(_context, _connectionState, _redis, channel, _formatter, _logger, _cachingTelemetryProvider, redisProfiler, _waiter, () => _subject.Invalidate(MissedEventsReason.Lost), _stopTokenSource.Token); _dispatcher = new EventDispatcher(topicKey, channel, _subject, _logger, _stopTokenSource.Token); } @@ -89,7 +91,17 @@ public IDisposable Subscribe(IObserver observer) { this.ThrowIfDisposed(_disposed); CreateConsumerGroup(); - return _subject.Subscribe(observer); + + // Marked first, so a loss reported next expires this observer too; one reported while it joined may have expired + // the others before it, so it is told itself. + var mark = _subscriber.MarkSubscribed(); + var subscription = _subject.Subscribe(observer); + if (_subscriber.LostSince(mark) && observer is IMissedEventsObserver missed) + { + TellMissed(missed, MissedEventsReason.Lost); + } + + return subscription; } public async ValueTask PublishAsync(T @event, CancellationToken token = default) @@ -160,6 +172,19 @@ public void Dispose() internal RedisStreamsTopicOptions GetResolvedOptionsForTests() => _streamOptions; + /// Isolated like the subject's own notifications, so a throwing observer cannot fail the subscribe that registered it. + private void TellMissed(IMissedEventsObserver missed, MissedEventsReason reason) + { + try + { + missed.OnEventsMissed(reason); + } + catch (Exception ex) + { + LogObserverMissedEventsFailed(ex, TopicKey); + } + } + private void CreateConsumerGroup() { if (!_consumerGroupCreated) @@ -184,10 +209,7 @@ private bool EnsureStreamGroup() { try { - return _redis.Database.StreamCreateConsumerGroup( - _context.Topic, - _context.ConsumerGroup, - StreamPosition.NewMessages); + return ConsumerGroups.Create(_redis.Database, _context.Topic, _context.ConsumerGroup, StreamPosition.NewMessages); } catch (RedisServerException ex) when (ex.Message == StreamConstants.ConsumerGroupNameExistsErrorMessage) { @@ -219,6 +241,9 @@ private RedisStreamContext GetContext( [LoggerMessage(Level = LogLevel.Debug, Message = "Published to topic {TopicKey} event {EventId} stream id {StreamId}")] private partial void LogPublished(TopicKey topicKey, string? eventId, RedisValue streamId); + [LoggerMessage(Level = LogLevel.Warning, Message = "Observer threw in OnEventsMissed on topic {TopicKey}; it stays subscribed.")] + private partial void LogObserverMissedEventsFailed(Exception ex, TopicKey topicKey); + [LoggerMessage(Level = LogLevel.Warning, Message = "Error when publishing to topic {TopicKey} event {EventId}")] private partial void LogPublishError(Exception ex, TopicKey topicKey, string? eventId); diff --git a/src/UiPath.Caching/Broadcast/Redis/StreamGap.cs b/src/UiPath.Caching/Broadcast/Redis/StreamGap.cs new file mode 100644 index 0000000..59ade2f --- /dev/null +++ b/src/UiPath.Caching/Broadcast/Redis/StreamGap.cs @@ -0,0 +1,7 @@ +namespace UiPath.Caching.Broadcast.Redis; + +/// +/// Entries removed past a group's last delivery: on Redis 7.0, by the newest entry deleted and the count removed; below +/// it, by the oldest entry left. An equal gap found later lost nothing more. +/// +internal readonly record struct StreamGap(RedisValue LastDelivered, RedisValue Evidence, long Removed); diff --git a/src/UiPath.Caching/Broadcast/Redis/StreamIds.cs b/src/UiPath.Caching/Broadcast/Redis/StreamIds.cs new file mode 100644 index 0000000..c1b502c --- /dev/null +++ b/src/UiPath.Caching/Broadcast/Redis/StreamIds.cs @@ -0,0 +1,125 @@ +using System.Globalization; + +namespace UiPath.Caching.Broadcast.Redis; + +internal static class StreamIds +{ + /// + /// The entries past the group's last delivery that were removed, if any. Redis 7.0 records the newest entry deleted + /// and counts the entries added and read; below it, an oldest entry past the last delivery is taken as a loss. + /// keeps the last read count found, for the checks where Redis cannot tell it. + /// + public static StreamGap? Gap(StreamInfo stream, StreamGroupInfo group, ref ReadCount? known) => + Gap( + stream.Length, + stream.Length == 0 ? stream.LastGeneratedId : stream.FirstEntry.Id, + stream.LastGeneratedId, + stream.MaxDeletedEntryId, + stream.EntriesAdded, + group.LastDeliveredId, + group.EntriesRead is > 0 ? group.EntriesRead : null, // StackExchange.Redis reads a count Redis cannot tell as 0. + ref known); + + /// The same, from the raw XINFO STREAM and XINFO GROUPS replies a script returned. + public static StreamGap? Gap(RedisResult stream, RedisResult groups, RedisValue groupName, ref ReadCount? known) + { + var info = stream.ToDictionary(); + var length = (long)info["length"]; + var lastGenerated = (RedisValue)info["last-generated-id"]; + var oldest = length == 0 ? lastGenerated : (RedisValue)Items(info["first-entry"])[0]; + var maxDeleted = info.TryGetValue("max-deleted-entry-id", out var deleted) ? (RedisValue)deleted : RedisValue.Null; + var added = info.TryGetValue("entries-added", out var entriesAdded) ? (long)entriesAdded : 0; + foreach (var group in Items(groups)) + { + var fields = group.ToDictionary(); + if ((RedisValue)fields["name"] != groupName) + { + continue; + } + + long? read = fields.TryGetValue("entries-read", out var entriesRead) && !entriesRead.IsNull ? (long)entriesRead : null; + return Gap(length, oldest, lastGenerated, maxDeleted, added, (RedisValue)fields["last-delivered-id"], read, ref known); + } + + return null; + } + + /// The entries of an XREADGROUP reply over one stream, none when it returned nil. + public static StreamEntry[] Entries(RedisResult reply) + { + if (reply.IsNull) + { + return []; + } + + var entries = Items(Items(Items(reply)[0])[1]); + var parsed = new StreamEntry[entries.Length]; + for (var i = 0; i < entries.Length; i++) + { + var entry = Items(entries[i]); + var pairs = Items(entry[1]); + var values = new NameValueEntry[pairs.Length / 2]; + for (var j = 0; j < values.Length; j++) + { + values[j] = new NameValueEntry((RedisValue)pairs[2 * j], (RedisValue)pairs[(2 * j) + 1]); + } + + parsed[i] = new StreamEntry((RedisValue)entry[0], values); + } + + return parsed; + } + + /// The elements of an array reply, none for nil. + public static RedisResult[] Items(RedisResult reply) => (RedisResult[]?)reply ?? []; + + private static StreamGap? Gap( + long length, + RedisValue oldest, + RedisValue lastGenerated, + RedisValue maxDeleted, + long entriesAdded, + RedisValue lastDelivered, + long? entriesRead, + ref ReadCount? known) + { + if (!TryParse(lastDelivered, out var delivered)) + { + return null; + } + + // Redis reports no count for a group created at $ until it reads, nor after it reads past a deleted entry. The count + // is then every entry added while the group is at the newest entry, or the one found while it was at the same entry. + if (entriesRead is null && lastDelivered == lastGenerated) + { + entriesRead = entriesAdded; + } + else if (entriesRead is null && known is { } last && last.LastDelivered == lastDelivered) + { + entriesRead = last.EntriesRead; + } + + known = entriesRead is { } count ? new ReadCount(lastDelivered, count) : null; + + var pastOldest = TryParse(oldest, out var first) && first.CompareTo(delivered) > 0; + if (!TryParse(maxDeleted, out var deleted)) + { + return pastOldest ? new StreamGap(lastDelivered, oldest, 0) : null; + } + + // XDEL records the entry it removed; a trim shows only in the counts, so with none known one is assumed. + var removed = entriesAdded - length; + var trimmed = pastOldest && (entriesRead is not { } read || entriesAdded - read > length); + return trimmed || deleted.CompareTo(delivered) > 0 ? new StreamGap(lastDelivered, maxDeleted, removed) : null; + } + + private static bool TryParse(RedisValue id, out (ulong Ms, ulong Seq) parsed) + { + parsed = default; + var text = (string?)id; + var dash = text?.IndexOf('-', StringComparison.Ordinal) ?? -1; + return dash > 0 + && ulong.TryParse(text.AsSpan(0, dash), NumberStyles.None, CultureInfo.InvariantCulture, out parsed.Ms) + && ulong.TryParse(text.AsSpan(dash + 1), NumberStyles.None, CultureInfo.InvariantCulture, out parsed.Seq); + } +} diff --git a/src/UiPath.Caching/CacheEventPublisher.cs b/src/UiPath.Caching/CacheEventPublisher.cs index 27b4082..f7486c3 100644 --- a/src/UiPath.Caching/CacheEventPublisher.cs +++ b/src/UiPath.Caching/CacheEventPublisher.cs @@ -62,7 +62,7 @@ public ValueTask CacheRefreshedAsync(ICacheEntryOptions options, Type? ent { [KnownFieldNames.ExpirationKey] = options.Expiration, }; - return RaiseEventAsync(options, KnownEventTypes.CacheRemoved, entryType, properties); + return RaiseEventAsync(options, KnownEventTypes.CacheRefreshed, entryType, properties); } public ValueTask CacheRemovedAsync(ICacheEntryOptions options) => diff --git a/src/UiPath.Caching/HashLocalMemorySetter.cs b/src/UiPath.Caching/HashLocalMemorySetter.cs index 494132c..cb2023f 100644 --- a/src/UiPath.Caching/HashLocalMemorySetter.cs +++ b/src/UiPath.Caching/HashLocalMemorySetter.cs @@ -10,8 +10,9 @@ internal sealed class HashLocalMemorySetter( IMultilayerCacheOptions cacheOptions, IMemoryCacheOptions memoryCacheOptions, Telemetry.ICachingTelemetryProvider telemetryProvider, - KeyMasker? masker = null) - : MemoryCacheSetter(cacheName, changeTokenFactory, topicProvider, memoryCache, logger, clock, cacheOptions, memoryCacheOptions, telemetryProvider, masker) + KeyMasker? masker = null, + bool ignoreSubscriptionGaps = false) + : MemoryCacheSetter(cacheName, changeTokenFactory, topicProvider, memoryCache, logger, clock, cacheOptions, memoryCacheOptions, telemetryProvider, masker, ignoreSubscriptionGaps) { protected override ICacheEntryOptions CreateEntry(RefreshMetadataState metadataState, CancellationToken cancellationToken) { diff --git a/src/UiPath.Caching/IInFlightRun.cs b/src/UiPath.Caching/IInFlightRun.cs new file mode 100644 index 0000000..01af5f4 --- /dev/null +++ b/src/UiPath.Caching/IInFlightRun.cs @@ -0,0 +1,10 @@ +namespace UiPath.Caching; + +/// The shared run as its work sees it: the token it runs on, and permission to commit a side effect only while a caller still waits. +internal interface IInFlightRun +{ + CancellationToken Token { get; } + + /// Runs unless every caller has left; false when it did not. + bool TryCommit(TArg arg, Action commit); +} diff --git a/src/UiPath.Caching/IMultilayerCacheOptions.cs b/src/UiPath.Caching/IMultilayerCacheOptions.cs index 6d5878e..65c0dde 100644 --- a/src/UiPath.Caching/IMultilayerCacheOptions.cs +++ b/src/UiPath.Caching/IMultilayerCacheOptions.cs @@ -13,9 +13,12 @@ public interface IMultilayerCacheOptions : ICacheOptions TimeSpan? ConnectionMonitorPeriod { get; set; } - /// Serve from L1 only (without falling back to default) when the L2 connection is unhealthy. Aligns with Local/Distributed tier naming. + /// Serve from L1 only (without falling back to default) when the L2 connection is unhealthy. Defaults to true; needs the connection monitor. bool? UseLocalOnlyWhenDisconnected { get; set; } + /// Clear L1 when a Redis Pub/Sub broadcast reconnects, since invalidations sent while it was down never arrived. Defaults to true; needs the connection monitor. + bool? ClearLocalOnReconnect { get; set; } + /// L1 cap on entry lifetime while the L2 connection is unhealthy (paired with ). TimeSpan? LocalMaxExpirationDisconnected { get; set; } diff --git a/src/UiPath.Caching/InFlight.cs b/src/UiPath.Caching/InFlight.cs new file mode 100644 index 0000000..6e207bc --- /dev/null +++ b/src/UiPath.Caching/InFlight.cs @@ -0,0 +1,164 @@ +using System.Collections.Concurrent; + +namespace UiPath.Caching; + +/// One run of the work per key at a time: concurrent callers for the same key join the run in progress and share its result or its failure. +/// The shared work runs on its own token, cancelled once every caller still waiting has cancelled; a caller that cancels stops waiting without cancelling it for the others. +internal sealed class InFlight + where TKey : notnull +{ + private readonly ConcurrentDictionary _flights = new(); + + /// Runs in the table, for tests. + internal int Count => _flights.Count; + + public async ValueTask RunAsync(TKey key, TState state, Func> work, CancellationToken token) + { + // A caller cancelled before it arrives starts no work and joins none. + token.ThrowIfCancellationRequested(); + var flight = Join(key, state, work); + if (!token.CanBeCanceled) + { + return await flight.Result.ConfigureAwait(false); + } + + try + { + return await flight.Result.WaitAsync(token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + // Left here rather than from a token callback: WaitAsync's own callback can resume this caller before a registration of ours would run. + flight.Leave(); + throw; + } + } + + private Flight Join(TKey key, TState state, Func> work) + { + while (true) + { + if (_flights.TryGetValue(key, out var running)) + { + if (running.TryJoin()) + { + return running; + } + + // Every caller has left it, so it may be cancelled: start a fresh one. + _flights.TryRemove(new KeyValuePair(key, running)); + continue; + } + + var started = new Flight(this, key); + if (_flights.TryAdd(key, started)) + { + _ = started.RunAsync(state, work); + return started; + } + + started.Dispose(); + } + } + + private sealed class Flight(InFlight owner, TKey key) : IInFlightRun, IDisposable + { + private readonly CancellationTokenSource _cancellation = new(); + private readonly TaskCompletionSource _result = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly object _gate = new(); + private int _waiting = 1; + + public Task Result => _result.Task; + + public CancellationToken Token => _cancellation.Token; + + /// False once every caller has left, since the work may already be cancelled. + public bool TryJoin() + { + lock (_gate) + { + if (_waiting == 0) + { + return false; + } + + _waiting++; + return true; + } + } + + public void Leave() + { + lock (_gate) + { + if (--_waiting > 0) + { + return; + } + + // Out of the table now, not when the work ends: work that ignores its token would otherwise hold the entry. + owner._flights.TryRemove(new KeyValuePair(key, this)); + } + + try + { + _cancellation.Cancel(); + } + catch (ObjectDisposedException) + { + // The work finished first. + } + } + + public bool TryCommit(TArg arg, Action commit) + { + // Under the gate Leave takes, so work every caller has left cannot commit after a fresh run has started. + lock (_gate) + { + if (_waiting == 0) + { + return false; + } + + commit(arg); + return true; + } + } + + public void Dispose() => _cancellation.Dispose(); + + public async Task RunAsync(TState state, Func> work) + { + TResult result = default!; + Exception? failure = null; + try + { + result = await work(state, this).ConfigureAwait(false); + } + catch (Exception ex) + { + failure = ex; + } + + // Out of the table before anyone sees the outcome, so a caller that arrives next starts a fresh run. + owner._flights.TryRemove(new KeyValuePair(key, this)); + if (failure is OperationCanceledException && _cancellation.IsCancellationRequested) + { + _result.TrySetCanceled(_cancellation.Token); + } + else if (failure is not null) + { + _result.TrySetException(failure); + + // Every caller may have left already, leaving nothing to observe the failure. + _result.Task.Forget(); + } + else + { + _result.TrySetResult(result); + } + + Dispose(); + } + } +} diff --git a/src/UiPath.Caching/InFlightKey.cs b/src/UiPath.Caching/InFlightKey.cs new file mode 100644 index 0000000..d655eaa --- /dev/null +++ b/src/UiPath.Caching/InFlightKey.cs @@ -0,0 +1,4 @@ +namespace UiPath.Caching; + +/// A composed key, the type read under it and the local lifetimes a hit is kept for, the parts of a policy a shared read uses: reads that differ in any of them are not shared. +internal readonly record struct InFlightKey(string Name, Type Type, TimeSpan? LocalExpiration, TimeSpan? LocalExpirationDisconnected); diff --git a/src/UiPath.Caching/InMemoryCacheOptions.cs b/src/UiPath.Caching/InMemoryCacheOptions.cs index 657589e..a4e7b31 100644 --- a/src/UiPath.Caching/InMemoryCacheOptions.cs +++ b/src/UiPath.Caching/InMemoryCacheOptions.cs @@ -40,6 +40,8 @@ public class InMemoryCacheOptions : IMultilayerCacheOptions, IMemoryCacheOptions public bool? UseLocalOnlyWhenDisconnected { get; set; } + public bool? ClearLocalOnReconnect { get; set; } + public TimeSpan? LocalMaxExpirationDisconnected { get; set; } = TimeSpan.FromSeconds(30); public bool? LocalLockEnabled { get; set; } = true; diff --git a/src/UiPath.Caching/InMemoryRedisCacheOptions.cs b/src/UiPath.Caching/InMemoryRedisCacheOptions.cs index 59bf44e..3ac924f 100644 --- a/src/UiPath.Caching/InMemoryRedisCacheOptions.cs +++ b/src/UiPath.Caching/InMemoryRedisCacheOptions.cs @@ -56,6 +56,8 @@ public class InMemoryRedisCacheOptions : IMultilayerCacheOptions, IMemoryCacheOp public bool? UseLocalOnlyWhenDisconnected { get; set; } + public bool? ClearLocalOnReconnect { get; set; } + public TimeSpan? LocalMaxExpirationDisconnected { get; set; } = TimeSpan.FromSeconds(30); public bool? LocalLockEnabled { get; set; } = true; diff --git a/src/UiPath.Caching/LocalMemorySetter.cs b/src/UiPath.Caching/LocalMemorySetter.cs index e36f5d3..8f69627 100644 --- a/src/UiPath.Caching/LocalMemorySetter.cs +++ b/src/UiPath.Caching/LocalMemorySetter.cs @@ -12,8 +12,9 @@ internal sealed class LocalMemorySetter( IMultilayerCacheOptions cacheOptions, IMemoryCacheOptions memoryCacheOptions, ICachingTelemetryProvider telemetryProvider, - KeyMasker? masker = null) - : MemoryCacheSetter(cacheName, changeTokenFactory, topicProvider, memoryCache, logger, clock, cacheOptions, memoryCacheOptions, telemetryProvider, masker) + KeyMasker? masker = null, + bool ignoreSubscriptionGaps = false) + : MemoryCacheSetter(cacheName, changeTokenFactory, topicProvider, memoryCache, logger, clock, cacheOptions, memoryCacheOptions, telemetryProvider, masker, ignoreSubscriptionGaps) { protected override ICacheEntryOptions CreateEntry(RefreshMetadataState metadataState, CancellationToken cancellationToken) { diff --git a/src/UiPath.Caching/MemoryCacheSetter.cs b/src/UiPath.Caching/MemoryCacheSetter.cs index f4414f1..0dcfc81 100644 --- a/src/UiPath.Caching/MemoryCacheSetter.cs +++ b/src/UiPath.Caching/MemoryCacheSetter.cs @@ -12,7 +12,8 @@ internal abstract class MemoryCacheSetter( IMultilayerCacheOptions cacheOptions, IMemoryCacheOptions memoryCacheOptions, ICachingTelemetryProvider telemetryProvider, - KeyMasker? masker = null + KeyMasker? masker = null, + bool ignoreSubscriptionGaps = false ) { @@ -22,6 +23,9 @@ internal abstract class MemoryCacheSetter( private const string PropTransportId = "TransportId"; private readonly KeyMasker _masker = masker ?? KeyMasker.Off; + // A custom factory cannot be told the cache keeps entries over a subscription gap, so it is handed a topic that hides one. + private readonly System.Runtime.CompilerServices.ConditionalWeakTable, ITopic> _gapFiltered = []; + protected TimeProvider Clock { get; } = clock; private ICacheEntrySizeProvider SizeProvider { get; } = memoryCacheOptions.SizeProvider ?? new DefaultCacheEntrySizeProvider(); @@ -32,8 +36,8 @@ public bool Set(ICacheEntryOptions options, ICacheEntry item, Type entryType, Ti { var topic = topicProvider.Create(options.TopicKey); var token = changeTokenFactory is IMaskedChangeTokenFactory masked - ? masked.Create(options.CacheKey, topic, cacheName, entryType, _masker, options.CallerKey) - : changeTokenFactory.Create(options.CacheKey, topic, cacheName, entryType); + ? masked.Create(options.CacheKey, topic, cacheName, entryType, _masker, options.CallerKey, ignoreSubscriptionGaps) + : changeTokenFactory.Create(options.CacheKey, TopicForCustomFactory(topic), cacheName, entryType); var state = new RefreshMetadataState(options.CacheKey, options.TopicKey, item, token, entryType, maxExpiration, options.CallerKey); token.RegisterChangeCallback(RefreshMetadata, state); // Filled in place rather than through MemoryCacheEntryOptions, which is copied into the entry and discarded. @@ -75,6 +79,9 @@ private static void PostEviction(object key, object? value, EvictionReason reaso } } + private ITopic TopicForCustomFactory(ITopic topic) => + ignoreSubscriptionGaps ? _gapFiltered.GetValue(topic, static t => new GapFilteringTopic(t)) : topic; + private void RefreshMetadata(RefreshMetadataState metadataState) { var token = metadataState.Token; diff --git a/src/UiPath.Caching/MultilayerCache.cs b/src/UiPath.Caching/MultilayerCache.cs index 677e02a..53bd88b 100644 --- a/src/UiPath.Caching/MultilayerCache.cs +++ b/src/UiPath.Caching/MultilayerCache.cs @@ -6,6 +6,7 @@ namespace UiPath.Caching; internal sealed partial class MultilayerCache : MultilayerCacheBase, ICache, ISpanKeyCache { private readonly ICache _innerCache; + private readonly InFlight _innerReads = new(); private readonly CacheEntryBuilder _entryBuilder; private readonly LocalMemorySetter _localMemorySetter; @@ -32,7 +33,7 @@ public MultilayerCache( var cacheKeyStrategy = _multiLayerCacheOptions.CacheKeyStrategy ?? new DefaultCacheKeyStrategy(); var topicKeyStrategy = _multiLayerCacheOptions.TopicKeyStrategy ?? new DefaultTopicKeyStrategy(cacheOptions.Separator); _entryBuilder = new CacheEntryBuilder(cacheKeyStrategy, topicKeyStrategy, _clock); - _localMemorySetter = new LocalMemorySetter(cacheName, changeTokenFactory, _topicProvider, _memoryCache, logger, _clock, _multiLayerCacheOptions, memoryCacheOptions, telemetryProvider, _masker); + _localMemorySetter = new LocalMemorySetter(cacheName, changeTokenFactory, _topicProvider, _memoryCache, logger, _clock, _multiLayerCacheOptions, memoryCacheOptions, telemetryProvider, _masker, !_clearLocalOnReconnect); } [OverloadResolutionPriority(1)] @@ -556,7 +557,7 @@ private async ValueTask WriteRehydrateGroupsAsync( foreach (var group in groups.Values) { var fired = innerCacheDisconnected || await PublishCacheSetEventsAsync(group).ConfigureAwait(false); - var written = fired && await InternalSetAsync(group.Select(e => e.Entry).ToArray(), innerCacheDisconnected, policy, token).ConfigureAwait(false); + var written = fired && await InternalSetAsync(group.Select(e => e.Entry).ToArray(), innerCacheDisconnected, policy, keepRefused: false, token).ConfigureAwait(false); if (!written) { failed.AddRange(group.Select(e => e.CallerKey.Name)); @@ -590,7 +591,7 @@ private async ValueTask PublishCacheSetEventsAsync(List<(CacheEntryValu if (ret is not null || _multiLayerCacheOptions.CacheNullValues) { var innerCacheDisconnected = GetInnerCacheDisconnected(); - await InternalSetAsync(cacheEntryOptions, ret, innerCacheDisconnected, policy).ConfigureAwait(false); + await InternalSetAsync(cacheEntryOptions, ret, innerCacheDisconnected, policy, keepRefused: true).ConfigureAwait(false); } return _cacheEntryFactory.Create(ret, cacheEntryOptions.Expiration); } @@ -724,7 +725,7 @@ private async ValueTask PublishCacheSetEventsAsync(List<(CacheEntryValu if (toStore.Count > 0) { var innerCacheDisconnected = GetInnerCacheDisconnected(); - await InternalSetAsync(toStore.ToArray(), innerCacheDisconnected, policy, token).ConfigureAwait(false); + await InternalSetAsync(toStore.ToArray(), innerCacheDisconnected, policy, keepRefused: true, token).ConfigureAwait(false); } return BuildBatchEntries(missOptions, missStates, probe, producedByState); @@ -958,7 +959,7 @@ private async ValueTask SetCoreAsync(KeyValuePair[] keyVa } var innerCacheDisconnected = GetInnerCacheDisconnected(); - var internalSetResult = await InternalSetAsync(setEntries.ToArray(), innerCacheDisconnected, policy, token).ConfigureAwait(false); + var internalSetResult = await InternalSetAsync(setEntries.ToArray(), innerCacheDisconnected, policy, keepRefused: false, token).ConfigureAwait(false); if (!internalSetResult) { return false; @@ -1110,7 +1111,7 @@ private async ValueTask RemoveAsync(CacheEntryOptions[] options, Cancel LogFoundInnerCacheCopy(LoggedComposed(key, typeof(T))); var option = cacheEntriesToFetch[i]; option.Expiration = entry.Expiration; - MemorySet(option, new KeyValuePair(key, entry.Value), policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); + MemorySet(option, entry.Value, LocalLifetime(policy)); } return results.ToArray(); @@ -1138,17 +1139,8 @@ private async ValueTask RemoveAsync(CacheEntryOptions[] options, Cancel } } - var fetched = await _innerCache.GetCacheEntryAsync(options.CacheKey, policy, options.Token).ConfigureAwait(false); - - if (!fetched.Found) - { - return default; - } - - LogFoundInnerCacheCopy(Logged(options, typeof(T))); - options.Expiration = fetched.Expiration; - MemorySet(options, fetched.Value, policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); - return fetched.Value; + var fetched = await FetchInnerAsync(options, policy).ConfigureAwait(false); + return fetched.Found ? fetched.Value : default; } private async ValueTask>[]> GetCacheEntriesInnerAsync(CacheEntryOptions[] options, CachePolicy policy, CancellationToken token = default) @@ -1205,7 +1197,7 @@ private async ValueTask RemoveAsync(CacheEntryOptions[] options, Cancel LogFoundInnerCacheCopy(LoggedComposed(key, typeof(T))); var option = cacheEntriesToFetch[j]; option.Expiration = entry.Expiration; - MemorySet(option, new KeyValuePair(key, entry.Value), policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); + MemorySet(option, entry.Value, LocalLifetime(policy)); } return results; @@ -1230,20 +1222,37 @@ private async ValueTask RemoveAsync(CacheEntryOptions[] options, Cancel return _cacheEntryFactory.Create(default, DateTimeOffset.MinValue); } - var fetched = await _innerCache.GetCacheEntryAsync(options.CacheKey, policy, options.Token).ConfigureAwait(false); + return await FetchInnerAsync(options, policy).ConfigureAwait(false); + } + + /// Reads the inner tier and keeps a hit locally; concurrent reads of one key share one inner read. + private async ValueTask> FetchInnerAsync(CacheEntryOptions options, CachePolicy policy) => + (ICacheEntry)(await _innerReads.RunAsync( + new InFlightKey(options.CacheKey.Name, typeof(T), policy.LocalExpiration, policy.LocalExpirationDisconnected), + (Cache: this, Options: options, Policy: policy), + static async (state, run) => await state.Cache.FetchAndKeepAsync(state.Options with { Token = run.Token }, state.Policy, run).ConfigureAwait(false), + options.Token).ConfigureAwait(false))!; - if (!fetched.Found) + private async ValueTask FetchAndKeepAsync(CacheEntryOptions options, CachePolicy policy, IInFlightRun run) + { + var fetched = await _innerCache.GetCacheEntryAsync(options.CacheKey, policy, options.Token).ConfigureAwait(false); + if (fetched.Found) { - return fetched; + // Committed only while a caller still waits, so a read every caller left cannot overwrite a fresh one. + run.TryCommit((Cache: this, Options: options, Policy: policy, Fetched: fetched), static s => s.Cache.Keep(s.Options, s.Policy, s.Fetched)); } + return fetched; + } + + private void Keep(CacheEntryOptions options, CachePolicy policy, ICacheEntry fetched) + { LogFoundInnerCacheCopy(Logged(options, typeof(T))); options.Expiration = fetched.Expiration; - MemorySet(options, fetched.Value, policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); - return fetched; + MemorySet(options, fetched.Value, LocalLifetime(policy)); } - private async ValueTask InternalSetAsync(CacheEntryOptions options, T? value, bool innerCacheDisconnected, CachePolicy policy) + private async ValueTask InternalSetAsync(CacheEntryOptions options, T? value, bool innerCacheDisconnected, CachePolicy policy, bool keepRefused = false) { try { @@ -1253,8 +1262,18 @@ private async ValueTask InternalSetAsync(CacheEntryOptions options, T? return MemorySet(options, value, policy.LocalExpirationDisconnected ?? _multiLayerCacheOptions.LocalMaxExpirationDisconnected); } - var ret = await _innerCache.SetAsync(options.CacheKey, value, options.Expiration, policy, options.Token).ConfigureAwait(false); - return ret && MemorySet(options, value, policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); + bool ret; + try + { + ret = await _innerCache.SetAsync(options.CacheKey, value, options.Expiration, policy, options.Token).ConfigureAwait(false); + } + catch (Exception ex) when (keepRefused && !(ex is OperationCanceledException && options.Token.IsCancellationRequested)) + { + LogInnerCacheSetError(ex, Logged(options, typeof(T))); + ret = false; + } + + return KeepAfterInnerWrite(ret, keepRefused, policy, (Cache: this, Options: options, Value: value), static (s, max) => s.Cache.MemorySet(s.Options, s.Value, max), options.Token); } catch (Exception ex) { @@ -1263,7 +1282,7 @@ private async ValueTask InternalSetAsync(CacheEntryOptions options, T? } } - private async ValueTask InternalSetAsync(CacheEntryValue[] cacheEntries, bool innerCacheDisconnected, CachePolicy policy, CancellationToken token = default) + private async ValueTask InternalSetAsync(CacheEntryValue[] cacheEntries, bool innerCacheDisconnected, CachePolicy policy, bool keepRefused, CancellationToken token) { try { @@ -1275,40 +1294,45 @@ private async ValueTask InternalSetAsync(CacheEntryValue[] cacheEntr { LogSettingLocalOnlyForCacheKeys(Logged(cacheEntries.Select(o => o.CacheEntry).ToArray(), typeof(T))); } - return MemSet(policy.LocalExpirationDisconnected ?? _multiLayerCacheOptions.LocalMaxExpirationDisconnected); + return MemorySet(cacheEntries, policy.LocalExpirationDisconnected ?? _multiLayerCacheOptions.LocalMaxExpirationDisconnected); + } + + bool set; + try + { + // One deadline for the batch, taken from the first entry — every entry in a batch is + // built from the same resolved expiration. With no entries there is nothing to date, + // so the write inherits the policy. + set = cacheEntries.Length > 0 + ? await _innerCache.SetAsync(cacheKeyValuePairs, cacheEntries[0].CacheEntry.Expiration, policy, token).ConfigureAwait(false) + : await _innerCache.SetAsync(cacheKeyValuePairs, policy, token).ConfigureAwait(false); + } + catch (Exception ex) when (keepRefused && !(ex is OperationCanceledException && token.IsCancellationRequested)) + { + LogBatchSetError(ex, cacheEntries); + set = false; } - // One deadline for the batch, taken from the first entry — every entry in a batch is - // built from the same resolved expiration. With no entries there is nothing to date, - // so the write inherits the policy. - var set = cacheEntries.Length > 0 - ? await _innerCache.SetAsync(cacheKeyValuePairs, cacheEntries[0].CacheEntry.Expiration, policy, token).ConfigureAwait(false) - : await _innerCache.SetAsync(cacheKeyValuePairs, policy, token).ConfigureAwait(false); - return set && MemSet(policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); + return KeepAfterInnerWrite(set, keepRefused, policy, (Cache: this, Entries: cacheEntries), static (s, max) => s.Cache.MemorySet(s.Entries, max), token); } catch (Exception ex) { - if (_logger.IsEnabled(LogLevel.Warning)) - { - LogInnerCacheSetKeysError(ex, Logged(cacheEntries.Select(o => o.CacheEntry).ToArray(), typeof(T))); - } + LogBatchSetError(ex, cacheEntries); return false; } + } - bool MemSet(TimeSpan? maxExpiration) + private void LogBatchSetError(Exception ex, CacheEntryValue[] cacheEntries) + { + if (_logger.IsEnabled(LogLevel.Warning)) { - foreach (var cacheEntry in cacheEntries) - { - var set = MemorySet(cacheEntry.CacheEntry, cacheEntry.Value, maxExpiration); - if (!set) - { - return false; - } - } - return true; + LogInnerCacheSetKeysError(ex, Logged(cacheEntries.Select(o => o.CacheEntry).ToArray(), typeof(T))); } } + private bool MemorySet(CacheEntryValue[] cacheEntries, TimeSpan? maxExpiration) => + Array.TrueForAll(cacheEntries, e => MemorySet(e.CacheEntry, e.Value, maxExpiration)); + private bool MemorySet(CacheEntryOptions options, T value, TimeSpan? maxExpiration) { var item = _cacheEntryFactory.Create(value, options.Expiration); diff --git a/src/UiPath.Caching/MultilayerCacheBase.cs b/src/UiPath.Caching/MultilayerCacheBase.cs index 7a7b175..c3f030a 100644 --- a/src/UiPath.Caching/MultilayerCacheBase.cs +++ b/src/UiPath.Caching/MultilayerCacheBase.cs @@ -1,11 +1,12 @@ using System.Runtime.CompilerServices; +using UiPath.Caching.Broadcast.Redis; using UiPath.Caching.Config; using UiPath.Caching.Locking; using UiPath.Caching.Telemetry; namespace UiPath.Caching; -public abstract class MultilayerCacheBase : IDisposable +public abstract partial class MultilayerCacheBase : IDisposable { protected readonly ILogger _logger; protected readonly IMemoryCache _memoryCache; @@ -17,6 +18,7 @@ public abstract class MultilayerCacheBase : IDisposable protected readonly IConnectionState _connectionState; protected readonly ITopicProvider _topicProvider; protected readonly bool _useLocalOnlyWhenDisconnected; + private protected readonly bool _clearLocalOnReconnect; private protected readonly RehydrationCoordinator _rehydrator; private protected readonly CachePolicy _defaultPolicy; private protected readonly KeyMasker _masker; @@ -83,7 +85,8 @@ protected MultilayerCacheBase( _eventPublisher = new CacheEventPublisher(cacheName, _topicProvider, cacheEventFactory, logger, _masker); var connectionMonitorEnabled = multiLayerCacheOptions.ConnectionMonitorEnabled ?? cacheOptions.ConnectionMonitorEnabled; _connectionState = connectionMonitorEnabled ? GetConnectionMonitor(innerCache, _topicProvider) : NullConnectionStateMonitor.Instance; - _useLocalOnlyWhenDisconnected = (multiLayerCacheOptions.UseLocalOnlyWhenDisconnected ?? false) && connectionMonitorEnabled; + _useLocalOnlyWhenDisconnected = (multiLayerCacheOptions.UseLocalOnlyWhenDisconnected ?? true) && connectionMonitorEnabled; + _clearLocalOnReconnect = (multiLayerCacheOptions.ClearLocalOnReconnect ?? true) && connectionMonitorEnabled; _localLock = localLock; _distributedLock = distributedLock; _lockKeyStrategy = multiLayerCacheOptions.LockKeyStrategy ?? new DefaultDistributedLockKeyStrategy(cacheOptions.Separator); @@ -215,6 +218,30 @@ protected virtual void Dispose(bool disposing) } } + /// How long a hit read from the inner tier stays local: the disconnected cap while the tier reads as down, perhaps only its broadcast. + private protected TimeSpan? LocalLifetime(CachePolicy policy) => + GetInnerCacheDisconnected() + ? policy.LocalExpirationDisconnected ?? _multiLayerCacheOptions.LocalMaxExpirationDisconnected + : policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration; + + /// Writes locally after an inner write: for the normal lifetime when the inner tier took it, and with for the disconnected cap when it refused it. + private protected bool KeepAfterInnerWrite(bool written, bool keepRefused, CachePolicy policy, TState state, Func memorySet, CancellationToken token) + { + if (written) + { + return memorySet(state, policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); + } + + // The Redis setters answer a canceled write with false, so the caller's token tells a refusal from a cancellation. + if (keepRefused && !token.IsCancellationRequested) + { + // Kept as a disconnected write is kept, so the callers waiting on the local lock reuse it. + memorySet(state, policy.LocalExpirationDisconnected ?? _multiLayerCacheOptions.LocalMaxExpirationDisconnected); + } + + return false; + } + /// The key as a log line should show it. Nothing is rendered unless the line is written. private protected LoggedKey Logged(CacheKey key, Type? valueType = null) => LoggedKey.For(_masker, key, valueType); @@ -318,5 +345,4 @@ private IConnectionState GetConnectionMonitor(params object[] connectionStates) var lst = connectionStates.OfType().ToArray(); return lst.Length == 0 ? NullConnectionStateMonitor.Instance : new ConnectionStateMonitor(Telemetry, _multiLayerCacheOptions.ConnectionMonitorPeriod ?? TimeSpan.FromSeconds(5), lst); } - } diff --git a/src/UiPath.Caching/MultilayerHashCache.cs b/src/UiPath.Caching/MultilayerHashCache.cs index ae50333..0d17ad3 100644 --- a/src/UiPath.Caching/MultilayerHashCache.cs +++ b/src/UiPath.Caching/MultilayerHashCache.cs @@ -7,6 +7,7 @@ namespace UiPath.Caching; internal sealed partial class MultilayerHashCache : MultilayerCacheBase, IHashCache, ISpanKeyHashCache { private readonly IHashCache _innerCache; + private readonly InFlight _innerReads = new(); private readonly HashCacheEntryBuilder _entryBuilder; private readonly HashLocalMemorySetter _localMemorySetter; @@ -33,7 +34,7 @@ public MultilayerHashCache( var cacheKeyStrategy = _multiLayerCacheOptions.CacheKeyStrategy ?? new DefaultCacheKeyStrategy(); var topicKeyStrategy = _multiLayerCacheOptions.TopicKeyStrategy ?? new DefaultTopicKeyStrategy(cacheOptions.Separator); _entryBuilder = new HashCacheEntryBuilder(cacheKeyStrategy, topicKeyStrategy, _clock); - _localMemorySetter = new HashLocalMemorySetter(cacheName, changeTokenFactory, _topicProvider, _memoryCache, logger, _clock, _multiLayerCacheOptions, memoryCacheOptions, telemetryProvider, _masker); + _localMemorySetter = new HashLocalMemorySetter(cacheName, changeTokenFactory, _topicProvider, _memoryCache, logger, _clock, _multiLayerCacheOptions, memoryCacheOptions, telemetryProvider, _masker, !_clearLocalOnReconnect); } [OverloadResolutionPriority(1)] @@ -469,7 +470,7 @@ private void TriggerHashRehydrate(CacheKey originalCacheKey, DateTimeOffset e if (!IsNullOrEmpty(ret) || _multiLayerCacheOptions.CacheNullValues) { var innerCacheDisconnected = GetInnerCacheDisconnected(); - await InternalSetAsync(cacheEntryOptions, ret ?? Empty(), innerCacheDisconnected, policy).ConfigureAwait(false); + await InternalSetAsync(cacheEntryOptions, ret ?? Empty(), innerCacheDisconnected, policy, keepRefused: true).ConfigureAwait(false); } return _cacheEntryFactory.Create>(ret ?? Empty(), cacheEntryOptions.Expiration, cacheEntryOptions.Metadata); } @@ -538,29 +539,52 @@ private async ValueTask RemoveAsync(InternalHashCacheEntryOptions optio } } - cacheEntry = await _innerCache.GetCacheEntryAsync(options.CacheKey, policy, options.Token).ConfigureAwait(false); - + cacheEntry = await FetchInnerAsync(options, policy).ConfigureAwait(false); if (!cacheEntry.Found) { return cacheEntry!; } - LogFoundInnerCopy(Logged(options, typeof(T))); options.Expiration = cacheEntry.Expiration; options.Metadata = cacheEntry.Metadata; - var values = cacheEntry.Value ?? Empty(); - MemorySet(options, values, policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); - return Filter(cacheEntry!, options); } + /// Reads the whole entry from the inner tier and keeps a hit locally; concurrent reads of one key share one inner read, and each caller filters its own fields. + private async ValueTask>> FetchInnerAsync(InternalHashCacheEntryOptions options, CachePolicy policy) => + (ICacheEntry>)(await _innerReads.RunAsync( + new InFlightKey(options.CacheKey.Name, typeof(T), policy.LocalExpiration, policy.LocalExpirationDisconnected), + (Cache: this, Options: options, Policy: policy), + static async (state, run) => await state.Cache.FetchAndKeepAsync(state.Options with { Token = run.Token }, state.Policy, run).ConfigureAwait(false), + options.Token).ConfigureAwait(false))!; + + private async ValueTask FetchAndKeepAsync(InternalHashCacheEntryOptions options, CachePolicy policy, IInFlightRun run) + { + var fetched = await _innerCache.GetCacheEntryAsync(options.CacheKey, policy, options.Token).ConfigureAwait(false); + if (fetched.Found) + { + // Committed only while a caller still waits, so a read every caller left cannot overwrite a fresh one. + run.TryCommit((Cache: this, Options: options, Policy: policy, Fetched: fetched), static s => s.Cache.Keep(s.Options, s.Policy, s.Fetched)); + } + + return fetched; + } + + private void Keep(InternalHashCacheEntryOptions options, CachePolicy policy, ICacheEntry> fetched) + { + LogFoundInnerCopy(Logged(options, typeof(T))); + options.Expiration = fetched.Expiration; + options.Metadata = fetched.Metadata; + MemorySet(options, fetched.Value ?? Empty(), LocalLifetime(policy)); + } + private bool MemorySet(InternalHashCacheEntryOptions options, IDictionary value, TimeSpan? maxExpiration) { var item = CreateEntry(value, options); return _localMemorySetter.Set(options, item, typeof(T), maxExpiration); } - private async ValueTask InternalSetAsync(InternalHashCacheEntryOptions options, IDictionary value, bool disconnected, CachePolicy policy) + private async ValueTask InternalSetAsync(InternalHashCacheEntryOptions options, IDictionary value, bool disconnected, CachePolicy policy, bool keepRefused = false) { try { @@ -570,8 +594,18 @@ private async ValueTask InternalSetAsync(InternalHashCacheEntryOptions return MemorySet(options, value, policy.LocalExpirationDisconnected ?? _multiLayerCacheOptions.LocalMaxExpirationDisconnected); } - var ret = await _innerCache.SetAsync(options.CacheKey, value, new HashCacheEntryOptions(options.Expiration, null, options.Metadata, options.SetOption), policy, options.Token).ConfigureAwait(false); - return ret && MemorySet(options, value, policy.LocalExpiration ?? _multiLayerCacheOptions.LocalMaxExpiration); + bool ret; + try + { + ret = await _innerCache.SetAsync(options.CacheKey, value, new HashCacheEntryOptions(options.Expiration, null, options.Metadata, options.SetOption), policy, options.Token).ConfigureAwait(false); + } + catch (Exception ex) when (keepRefused && !(ex is OperationCanceledException && options.Token.IsCancellationRequested)) + { + LogInnerCacheSetError(ex, Logged(options, typeof(T))); + ret = false; + } + + return KeepAfterInnerWrite(ret, keepRefused, policy, (Cache: this, Options: options, Value: value), static (s, max) => s.Cache.MemorySet(s.Options, s.Value, max), options.Token); } catch (Exception ex) { diff --git a/src/UiPath.Caching/PublicAPI.Unshipped.txt b/src/UiPath.Caching/PublicAPI.Unshipped.txt index 34de8fc..a650ab9 100644 --- a/src/UiPath.Caching/PublicAPI.Unshipped.txt +++ b/src/UiPath.Caching/PublicAPI.Unshipped.txt @@ -1,4 +1,10 @@ #nullable enable +UiPath.Caching.Broadcast.IEventSubject.Invalidate(UiPath.Caching.Broadcast.MissedEventsReason reason) -> void +UiPath.Caching.Broadcast.IMissedEventsObserver +UiPath.Caching.Broadcast.IMissedEventsObserver.OnEventsMissed(UiPath.Caching.Broadcast.MissedEventsReason reason) -> void +UiPath.Caching.Broadcast.MissedEventsReason +UiPath.Caching.Broadcast.MissedEventsReason.Lost = 0 -> UiPath.Caching.Broadcast.MissedEventsReason +UiPath.Caching.Broadcast.MissedEventsReason.SubscriptionGap = 1 -> UiPath.Caching.Broadcast.MissedEventsReason UiPath.Caching.Redis.RedisCacheBase.SupportsExpireTime.get -> bool *REMOVED*UiPath.Caching.Redis.PrefixRedisKeyStrategy.Prefix.set -> void *REMOVED*UiPath.Caching.Redis.PrefixRedisKeyStrategy.Separator.set -> void @@ -118,3 +124,9 @@ UiPath.Caching.HashCache.GetCacheEntryAsync(System.Span cacheKey, Syste UiPath.Caching.HashCache.GetOrAddAsync(System.Span cacheKey, System.Func!>!>! generator, System.DateTimeOffset expiration, System.Threading.CancellationToken token = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask!> UiPath.Caching.HashCache.GetOrAddAsync(System.Span cacheKey, System.Func!>!>! generator, System.Threading.CancellationToken token = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask!> UiPath.Caching.HashCache.GetOrAddAsync(System.Span cacheKey, System.Func!>!>! generator, System.TimeSpan expiration, System.Threading.CancellationToken token = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask!> +UiPath.Caching.IMultilayerCacheOptions.ClearLocalOnReconnect.get -> bool? +UiPath.Caching.IMultilayerCacheOptions.ClearLocalOnReconnect.set -> void +UiPath.Caching.InMemoryCacheOptions.ClearLocalOnReconnect.get -> bool? +UiPath.Caching.InMemoryCacheOptions.ClearLocalOnReconnect.set -> void +UiPath.Caching.InMemoryRedisCacheOptions.ClearLocalOnReconnect.get -> bool? +UiPath.Caching.InMemoryRedisCacheOptions.ClearLocalOnReconnect.set -> void diff --git a/src/UiPath.Caching/Redis/ConnectionStateMonitor.cs b/src/UiPath.Caching/Redis/ConnectionStateMonitor.cs index 7ad3d0c..25f7c65 100644 --- a/src/UiPath.Caching/Redis/ConnectionStateMonitor.cs +++ b/src/UiPath.Caching/Redis/ConnectionStateMonitor.cs @@ -13,19 +13,40 @@ public sealed class ConnectionStateMonitor : IConnectionState, IDisposable private const string PropNow = "Now"; private const string PropConnected = "connected"; + private static readonly TimeSpan DefaultMonitorInterval = TimeSpan.FromSeconds(5); + private readonly IConnectionState[] _connectionStates; private readonly ICachingTelemetryProvider _telemetryProvider; + private readonly TimeProvider _timeProvider; + // Guards the outage mark, the observed flag and the timer, so failures, restores and polling change them one at a time. + // Never held while reading _isConnected: a Lazy holds its own lock while it evaluates, and its factory marks outages. + private readonly object _gate = new(); private Lazy _isConnected = default!; - private Timer? _timer; + private ITimer? _timer; + private bool _disposed; private TimeSpan _monitorInterval; + private bool _down; + private bool _observed; + private EventHandler? _recovered; public ConnectionStateMonitor( ICachingTelemetryProvider telemetryProvider, TimeSpan monitorInterval, params IConnectionState[] connectionStates) + : this(telemetryProvider, monitorInterval, TimeProvider.System, connectionStates) + { + } + + internal ConnectionStateMonitor( + ICachingTelemetryProvider telemetryProvider, + TimeSpan monitorInterval, + TimeProvider timeProvider, + params IConnectionState[] connectionStates) { _telemetryProvider = telemetryProvider; - _monitorInterval = monitorInterval; + _timeProvider = timeProvider; + // A zero period makes a one-shot timer, which could miss a recovery for good; infinite stays the explicit opt-out. + _monitorInterval = monitorInterval == TimeSpan.Zero ? DefaultMonitorInterval : monitorInterval; _connectionStates = connectionStates; ResetIsConnected(); foreach (var connectionState in _connectionStates) @@ -42,6 +63,26 @@ public ConnectionStateMonitor( public event EventHandler? OnReconnected; + /// Raised when every monitored state is connected again after an outage. + internal event EventHandler? Recovered + { + add + { + lock (_gate) + { + _recovered += value; + } + } + + remove + { + lock (_gate) + { + _recovered -= value; + } + } + } + public bool IsConnected => _isConnected.Value; public void Dispose() @@ -52,14 +93,31 @@ public void Dispose() connectionState.OnConnectionRestored -= InternalOnConnectionRestored; connectionState.OnReconnected -= InternalOnReconnected; } - _timer?.Dispose(); + + lock (_gate) + { + _disposed = true; + _timer?.Dispose(); + _timer = null; + } } private void InternalOnConnectionRestored(object? sender, EventArgs e) { TrackEvent(EventConnectionRestored); + lock (_gate) + { + // Before the aggregate was ever observed, a restore is the only sign of an outage that began before this monitor. + // After that, failures and evaluations mark outages, so a restore forwarded twice by two facades clears once. + if (!_observed) + { + _down = true; + } + } + ResetIsConnected(); OnConnectionRestored.TryRaise(_telemetryProvider, handler => handler(this, EventArgs.Empty)); + RaiseIfRecovered(); } private void InternalOnConnectionFailed(object? sender, EventArgs e) @@ -68,6 +126,11 @@ private void InternalOnConnectionFailed(object? sender, EventArgs e) TrackEvent(e is ConnectionFailedEventArgs { FailureType: ConnectionFailureType.MaintenanceHandoff } ? EventMaintenanceHandoff : EventConnectionFailed); + lock (_gate) + { + Accept(false); + } + ResetIsConnected(); OnConnectionFailed.TryRaise(_telemetryProvider, handler => handler(sender, e)); } @@ -75,36 +138,123 @@ private void InternalOnConnectionFailed(object? sender, EventArgs e) private void InternalOnReconnected(object? sender, EventArgs e) { TrackEvent(EventReconnected); + lock (_gate) + { + // A forced swap can drop publications without ever reading as down. One swap heard through two facades + // recovers twice in a row; the second clear finds the local tier already empty. + _down = true; + } + ResetIsConnected(); OnReconnected.TryRaise(_telemetryProvider, handler => handler(sender, e)); + RaiseIfRecovered(); } + private void ResetIsConnected(bool addTimer = true) { - _isConnected = new Lazy(() => { + Lazy isConnected = null!; + isConnected = new Lazy(() => { var ret = Array.TrueForAll(_connectionStates, static x => x.IsConnected); + lock (_gate) + { + // A replaced value may have been read before a restore that has since raised Recovered; marking it would raise it twice. + if (ReferenceEquals(_isConnected, isConnected)) + { + Accept(ret); + } + } + TrackEvent(EventEvaluateConnected, new KeyValuePair(PropConnected, ret.ToString())); return ret; }); + _isConnected = isConnected; - if (addTimer) + if (!addTimer) { - _timer = new Timer(_ => EvaluateConnected(), null, _monitorInterval, _monitorInterval); + return; } - } - private void EvaluateConnected() - { - if (_isConnected.Value) + lock (_gate) { + if (_disposed) + { + return; + } + + // Each failure or restore replaces the timer: stop the one it replaces, or it keeps polling for good. + // Each callback knows its own timer, so a late callback of a replaced one cannot stop its successor. + // Created disarmed and armed once published, so even a zero interval cannot fire before the callback has its timer. _timer?.Dispose(); - _timer = null; + ITimer timer = null!; + timer = _timeProvider.CreateTimer(_ => EvaluateConnected(timer), null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + _timer = timer; + timer.Change(_monitorInterval, _monitorInterval); } - else + } + + private void EvaluateConnected(ITimer caller) + { + var connected = AllConnected(); + lock (_gate) { + if (!ReferenceEquals(_timer, caller)) + { + // Replaced or disposed since it fired: its successor, if any, polls now, and this sample may predate a restore. + caller.Dispose(); + return; + } + + // Confirmed under the gate before the timer stops: a state that dropped since the sample, with no event to arm + // another timer, keeps this one polling. + connected = connected && AllConnected(); + + // A poll that finds the tier down marks the outage too, or a monitor started during one would never see it end. + Accept(connected); ResetIsConnected(false); + + // Stopped only once recovery holds: a state that dropped again before the final check keeps this one polling. + if (connected && RaiseIfRecovered()) + { + _timer = null; + caller.Dispose(); + } } } + /// Whether no outage is left to recover from. + private bool RaiseIfRecovered() + { + // Decided and raised under the gate, so a failure cannot slip between the check and the notification. + lock (_gate) + { + if (!_down) + { + return true; + } + + var connected = AllConnected(); + Accept(connected); + if (!connected) + { + return false; + } + + _down = false; + _recovered.TryRaise(_telemetryProvider, handler => handler(this, EventArgs.Empty)); + return true; + } + } + + /// The states themselves, read without evaluating the cached , which would fix its value earlier than its readers expect. + private bool AllConnected() => Array.TrueForAll(_connectionStates, static x => x.IsConnected); + + /// Records a sample the monitor acts on, under the gate: only then does a restore stop being the outage's sole sign, so one read meanwhile is not lost. + private void Accept(bool connected) + { + _observed = true; + _down |= !connected; + } + private void TrackEvent(string eventName, params KeyValuePair[] data) { var properties = new KeyValuePair[data.Length + 1]; diff --git a/src/UiPath.Caching/Redis/RedisConnector.cs b/src/UiPath.Caching/Redis/RedisConnector.cs index 5ceee09..116ae72 100644 --- a/src/UiPath.Caching/Redis/RedisConnector.cs +++ b/src/UiPath.Caching/Redis/RedisConnector.cs @@ -145,10 +145,12 @@ public bool IsConnected { get { + // Down only when a multiplexer exists and says so; it reconnects on its own. Until one exists, whether the first + // connect has not started, is pending or faulted, a command must go through: it waits for the connect or retries it. var lazy = _lazyCacheConnectionMultiplexer; - return lazy.IsValueCreated - && lazy.Value.IsCompletedSuccessfully - && lazy.Value.Result.IsConnected; + return !_disposed && (!lazy.IsValueCreated + || !lazy.Value.IsCompletedSuccessfully + || lazy.Value.Result.IsConnected); } } diff --git a/tests/UiPath.Caching.Tests/Broadcast/ChangeTokenTests.cs b/tests/UiPath.Caching.Tests/Broadcast/ChangeTokenTests.cs index 0560ef2..7dfdc54 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/ChangeTokenTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/ChangeTokenTests.cs @@ -153,6 +153,73 @@ public void OnError() callbackCalled.Should().BeTrue(); } + [Fact] + public void OnEventsMissed_expires_the_entry() + { + var callbackCalled = false; + Sut.RegisterChangeCallback(_ => callbackCalled = true, null); + + ((IMissedEventsObserver)Sut).OnEventsMissed(MissedEventsReason.Lost); + + Sut.HasChanged.Should().BeTrue(); + callbackCalled.Should().BeTrue(); + } + + [Theory] + [InlineData(false, MissedEventsReason.SubscriptionGap, true)] + [InlineData(true, MissedEventsReason.SubscriptionGap, false)] + [InlineData(true, MissedEventsReason.Lost, true)] + public void A_token_that_ignores_subscription_gaps_still_expires_on_a_known_loss(bool ignoreSubscriptionGaps, MissedEventsReason reason, bool changed) + { + var sut = new ChangeToken(_key, _topic, _source, _serializer, _fixture.Freeze>>(), _telemetryProvider, _acceptedEvents, KeyMasker.Off, entryType: null, callerKey: _key, ignoreSubscriptionGaps); + + ((IMissedEventsObserver)sut).OnEventsMissed(reason); + + sut.HasChanged.Should().Be(changed); + } + + [Fact] + public void A_callback_registered_after_the_token_changed_runs_at_once() + { + ((IMissedEventsObserver)Sut).OnEventsMissed(MissedEventsReason.Lost); + var called = false; + + Sut.RegisterChangeCallback(_ => called = true, null); + + called.Should().BeTrue(); + } + + [Fact] + public async Task Concurrent_notifications_and_registrations_do_not_collide() + { + var calls = 0; + using var start = new Barrier(3); + void Register() + { + start.SignalAndWait(TestContext.Current.CancellationToken); + for (var i = 0; i < 20_000; i++) + { + Sut.RegisterChangeCallback(_ => Interlocked.Increment(ref calls), null); + } + } + + void Notify() + { + start.SignalAndWait(TestContext.Current.CancellationToken); + for (var i = 0; i < 200; i++) + { + ((IMissedEventsObserver)Sut).OnEventsMissed(MissedEventsReason.Lost); + } + } + + var register = Task.Run(Register, TestContext.Current.CancellationToken); + var notify = Task.Run(Notify, TestContext.Current.CancellationToken); + + start.SignalAndWait(TestContext.Current.CancellationToken); + + await FluentActions.Awaiting(() => Task.WhenAll(register, notify)).Should().NotThrowAsync(); + } + [Fact] public void AcceptedEvents() { @@ -176,6 +243,98 @@ public void AcceptedEvents() Sut.HasChanged.Should().BeTrue(); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public void A_removal_or_a_loss_after_a_refresh_leaves_nothing_to_refresh_with(bool loss) + { + Sut.OnNext(new TestCacheEvent + { + Id = Guid.NewGuid().ToString(), + Source = new Uri("urn:machine"), + Data = new CacheEventData(_key, new Dictionary + { + ["_expiration_"] = DateTimeOffset.Now, + ["_metadata_"] = new Dictionary { ["key"] = _key }, + }), + Type = _fixture.Create(), + }); + + if (loss) + { + ((IMissedEventsObserver)Sut).OnEventsMissed(MissedEventsReason.Lost); + } + else + { + Sut.OnNext(new TestCacheEvent { Id = Guid.NewGuid().ToString(), Source = new Uri("urn:machine"), Data = new CacheEventData(_key), Type = _fixture.Create() }); + } + + Sut.HasChanged.Should().BeTrue(); + Sut.MetadataHasChanged.Should().BeFalse("the refresh callback would otherwise put the old value back"); + Sut.Expiration.Should().BeNull(); + Sut.Metadata.Should().BeNull(); + } + + [Theory] + [InlineData("loss")] + [InlineData("gap")] + [InlineData("removal")] + [InlineData("error")] + public void A_refresh_delivered_after_a_removal_or_a_loss_leaves_nothing_to_refresh_with(string expiry) + { + var refreshable = new List(); + Sut.RegisterChangeCallback(_ => refreshable.Add(Sut.MetadataHasChanged || Sut.Expiration is not null), null); + switch (expiry) + { + case "loss": + ((IMissedEventsObserver)Sut).OnEventsMissed(MissedEventsReason.Lost); + break; + case "gap": + ((IMissedEventsObserver)Sut).OnEventsMissed(MissedEventsReason.SubscriptionGap); + break; + case "removal": + Sut.OnNext(new TestCacheEvent { Id = Guid.NewGuid().ToString(), Source = new Uri("urn:machine"), Data = new CacheEventData(_key), Type = _fixture.Create() }); + break; + default: + Sut.OnError(_fixture.Create()); + break; + } + + // The dispatcher already held this observer when the loss was reported. + Sut.OnNext(new TestCacheEvent + { + Id = Guid.NewGuid().ToString(), + Source = new Uri("urn:machine"), + Data = new CacheEventData(_key, new Dictionary + { + ["_expiration_"] = DateTimeOffset.Now.AddMinutes(5), + ["_metadata_"] = new Dictionary { ["key"] = _key }, + }), + Type = _fixture.Create(), + }); + + refreshable.Should().Equal([false, false], "the refresh callback would otherwise put the old value back"); + Sut.Expiration.Should().BeNull(); + Sut.Metadata.Should().BeNull(); + } + + [Fact] + public void Two_refreshes_both_carry_their_expiration() + { + foreach (var minutes in new[] { 5, 10 }) + { + Sut.OnNext(new TestCacheEvent + { + Id = Guid.NewGuid().ToString(), + Source = new Uri("urn:machine"), + Data = new CacheEventData(_key, new Dictionary { ["_expiration_"] = DateTimeOffset.UnixEpoch.AddMinutes(minutes) }), + Type = _fixture.Create(), + }); + } + + Sut.Expiration.Should().Be(DateTimeOffset.UnixEpoch.AddMinutes(10), "only an expiry ends refreshing"); + } + [Fact] public void Events_with_extended_data() { diff --git a/tests/UiPath.Caching.Tests/Broadcast/ChannelHelperTests.cs b/tests/UiPath.Caching.Tests/Broadcast/ChannelHelperTests.cs index 3462604..4d6668c 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/ChannelHelperTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/ChannelHelperTests.cs @@ -2,6 +2,21 @@ namespace UiPath.Caching.Tests.Broadcast; public class ChannelHelperTests { + [Theory] + [InlineData(System.Threading.Channels.BoundedChannelFullMode.DropWrite, 2)] + [InlineData(System.Threading.Channels.BoundedChannelFullMode.DropOldest, 1)] + [InlineData(System.Threading.Channels.BoundedChannelFullMode.DropNewest, 1)] + public void A_full_bounded_channel_reports_what_it_drops(System.Threading.Channels.BoundedChannelFullMode mode, int expected) + { + var dropped = new List(); + var channel = ChannelHelper.Create(unbounded: false, capacity: 1, mode, dropped.Add); + + channel.Writer.TryWrite(1).Should().BeTrue(); + channel.Writer.TryWrite(2).Should().BeTrue(); + + dropped.Should().Equal(expected); + } + [Theory] [InlineData(10, 100, 100)] [InlineData(2048, 4096, 4096)] diff --git a/tests/UiPath.Caching.Tests/Broadcast/DroppedEventsTests.cs b/tests/UiPath.Caching.Tests/Broadcast/DroppedEventsTests.cs new file mode 100644 index 0000000..cbe7adf --- /dev/null +++ b/tests/UiPath.Caching.Tests/Broadcast/DroppedEventsTests.cs @@ -0,0 +1,115 @@ +using Microsoft.Extensions.Logging.Abstractions; + +namespace UiPath.Caching.Tests.Broadcast; + +public class DroppedEventsTests +{ + [Fact] + public async Task Drops_while_an_expiry_runs_are_folded_into_one_more() + { + var runs = 0; + using var running = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + void Expire() + { + if (Interlocked.Increment(ref runs) == 1) + { + running.Set(); + release.Wait(TimeSpan.FromSeconds(10)); + } + } + + var sut = new DroppedEvents("topic", Expire, NullLogger.Instance); + + sut.Dropped(); + running.Wait(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken).Should().BeTrue(); + for (var i = 0; i < 100; i++) + { + sut.Dropped(); + } + + release.Set(); + (await Eventually(() => Volatile.Read(ref runs) >= 2)).Should().BeTrue(); + await Task.Delay(200, TestContext.Current.CancellationToken); + + Volatile.Read(ref runs).Should().Be(2, "the drops during the first expiry need one more, not one each"); + } + + [Fact] + public async Task A_drop_during_an_expiry_waits_for_it_rather_than_running_alongside() + { + var inside = 0; + var most = 0; + using var release = new ManualResetEventSlim(); + void Expire() + { + var now = Interlocked.Increment(ref inside); + InterlockedMax(ref most, now); + release.Wait(TimeSpan.FromSeconds(10)); + Interlocked.Decrement(ref inside); + } + + var sut = new DroppedEvents("topic", Expire, NullLogger.Instance); + sut.Dropped(); + (await Eventually(() => Volatile.Read(ref inside) == 1)).Should().BeTrue(); + sut.Dropped(); + sut.Dropped(); + await Task.Delay(200, TestContext.Current.CancellationToken); + var duringFirst = Volatile.Read(ref most); + release.Set(); + await Task.Delay(200, TestContext.Current.CancellationToken); + + duringFirst.Should().Be(1, "the rerun waits for the expiry in progress"); + Volatile.Read(ref most).Should().Be(1); + } + + [Fact] + public async Task An_expiry_that_throws_is_logged_and_the_next_drop_still_expires() + { + var runs = 0; + void Expire() + { + Interlocked.Increment(ref runs); + throw new InvalidOperationException("boom"); + } + + var sut = new DroppedEvents("topic", Expire, NullLogger.Instance); + + sut.Dropped(); + (await Eventually(() => Volatile.Read(ref runs) == 1)).Should().BeTrue(); + sut.Dropped(); + + (await Eventually(() => Volatile.Read(ref runs) == 2)).Should().BeTrue("a throwing expiry must not leave the next drop pending for good"); + } + + private static async Task Eventually(Func condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline) + { + if (condition()) + { + return true; + } + + await Task.Delay(10, TestContext.Current.CancellationToken); + } + + return false; + } + + private static void InterlockedMax(ref int target, int value) + { + var current = Volatile.Read(ref target); + while (value > current) + { + var seen = Interlocked.CompareExchange(ref target, value, current); + if (seen == current) + { + return; + } + + current = seen; + } + } +} diff --git a/tests/UiPath.Caching.Tests/Broadcast/EventDispatcherTests.cs b/tests/UiPath.Caching.Tests/Broadcast/EventDispatcherTests.cs index 1e98fcc..64e8ab6 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/EventDispatcherTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/EventDispatcherTests.cs @@ -87,6 +87,7 @@ public void OnNext(ICacheEvent value) } public void OnCompleted() { } + public void Invalidate(MissedEventsReason reason) { } public void Dispose() { } private sealed class Disposable : IDisposable diff --git a/tests/UiPath.Caching.Tests/Broadcast/KeyedSubjectTests.cs b/tests/UiPath.Caching.Tests/Broadcast/KeyedSubjectTests.cs index 4385045..6b20c1c 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/KeyedSubjectTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/KeyedSubjectTests.cs @@ -492,6 +492,68 @@ public async Task High_volume_subscribe_unsubscribe_completes_without_error() "no worker may fault under concurrent subscribe, dispatch and dispose"); } + [Fact] + public void Invalidate_notifies_every_missed_events_observer_and_keeps_them_subscribed() + { + var keyed = new TestKeyedObserver("myKey"); + var broadcast = new TestBroadcastObserver(); + var plain = Substitute.For>(); + _sut.Subscribe(keyed); + _sut.Subscribe(broadcast); + _sut.Subscribe(plain); + + _sut.Invalidate(MissedEventsReason.Lost); + _sut.OnNext(CreateEvent("myKey")); + + keyed.Invalidations.Should().Be(1); + broadcast.Invalidations.Should().Be(1); + plain.DidNotReceive().OnError(Arg.Any()); + keyed.Events.Should().ContainSingle(); + keyed.Completed.Should().BeFalse(); + } + + [Fact] + public void Invalidate_continues_past_an_observer_that_throws() + { + var keyed = new TestKeyedObserver("myKey"); + _sut.Subscribe(new ThrowingBroadcastObserver()); + _sut.Subscribe(keyed); + + _sut.Invalidate(MissedEventsReason.Lost); + + keyed.Invalidations.Should().Be(1); + } + + [Fact] + public async Task A_subscriber_racing_completion_is_completed_exactly_once() + { + for (var round = 0; round < 50; round++) + { + var sut = new KeyedSubject(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + var observers = new ConcurrentBag(); + using var start = new Barrier(5); + void SubscribeMany() + { + start.SignalAndWait(TestContext.Current.CancellationToken); + for (var i = 0; i < 20_000; i++) + { + var observer = new TestKeyedObserver($"key{i % 16}"); + observers.Add(observer); + sut.Subscribe(observer); + } + } + + var subscribers = Enumerable.Range(0, 4).Select(_ => OnDedicatedThread(SubscribeMany, TestContext.Current.CancellationToken)).ToArray(); + + start.SignalAndWait(TestContext.Current.CancellationToken); + Thread.SpinWait(20_000); + sut.OnCompleted(); + await Task.WhenAll(subscribers); + + observers.Should().OnlyContain(o => o.Completions == 1, "round {0}: completion and a late subscriber may both see it, but only one completes it", round); + } + } + private static ICacheEvent CreateEvent(string? key) { return new TestCacheEvent @@ -509,25 +571,40 @@ private static Task OnDedicatedThread(Action action, CancellationToken token) => private static Task OnDedicatedThread(Func function, CancellationToken token) => Task.Factory.StartNew(function, token, TaskCreationOptions.LongRunning, TaskScheduler.Default); - private sealed class TestKeyedObserver(string key) : IKeyedObserver + private sealed class TestKeyedObserver(string key) : IKeyedObserver, IMissedEventsObserver { + private int _completions; + public string Key { get; } = key; public List Events { get; } = []; public bool Completed { get; private set; } + public int Invalidations { get; private set; } + + public int Completions => Volatile.Read(ref _completions); + public void OnNext(ICacheEvent value) => Events.Add(value); public void OnError(Exception error) { } - public void OnCompleted() => Completed = true; + public void OnCompleted() + { + Completed = true; + Interlocked.Increment(ref _completions); + } + + public void OnEventsMissed(MissedEventsReason reason) => Invalidations++; } - private sealed class TestBroadcastObserver : IObserver + private sealed class TestBroadcastObserver : IObserver, IMissedEventsObserver { public List Events { get; } = []; public bool Completed { get; private set; } + public int Invalidations { get; private set; } + public void OnNext(ICacheEvent value) => Events.Add(value); public void OnError(Exception error) { } public void OnCompleted() => Completed = true; + public void OnEventsMissed(MissedEventsReason reason) => Invalidations++; } private sealed class NoOpKeyedObserver(string key) : IKeyedObserver @@ -546,11 +623,12 @@ public void OnError(Exception error) { } public void OnCompleted() => throw new InvalidOperationException("boom on completed"); } - private sealed class ThrowingBroadcastObserver : IObserver + private sealed class ThrowingBroadcastObserver : IObserver, IMissedEventsObserver { public void OnNext(ICacheEvent value) => throw new InvalidOperationException("boom on next"); public void OnError(Exception error) { } public void OnCompleted() => throw new InvalidOperationException("boom on completed"); + public void OnEventsMissed(MissedEventsReason reason) => throw new InvalidOperationException("boom on events missed"); } private sealed class DelayingKeyedObserver(string key, TimeSpan delay) : IKeyedObserver diff --git a/tests/UiPath.Caching.Tests/Broadcast/RedisPubSubSubjectWriterTests.cs b/tests/UiPath.Caching.Tests/Broadcast/RedisPubSubSubjectWriterTests.cs index 45db95d..60182e7 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/RedisPubSubSubjectWriterTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/RedisPubSubSubjectWriterTests.cs @@ -1,3 +1,4 @@ +using System.Net; using System.Text; using System.Text.Json; using System.Threading.Channels; @@ -57,6 +58,174 @@ public async Task Receive_event(bool valid) _channel.Reader.TryRead(out var item).Should().Be(valid); } + [Fact] + public async Task A_resubscribe_after_a_reconnect_invalidates_and_the_first_subscribe_does_not() + { + var redis = _fixture.Freeze(); + redis.Subscriber.Returns(_subscriber); + var resubscribed = 0; + _fixture.Inject(() => Interlocked.Increment(ref resubscribed)); + var subscribes = 0; + _subscriber.When(x => x.Subscribe(_redisChannel, Arg.Any>())) + .Do(_ => Interlocked.Increment(ref subscribes)); + + Sut(); + await WaitForSubscribeAsync(); + await Task.Delay(_delay.Multiply(3), testContextAccessor.Current.CancellationToken); + Volatile.Read(ref resubscribed).Should().Be(0, "nothing was cached before the first subscription"); + + redis.OnReconnected += Raise.Event(redis, EventArgs.Empty); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline && Volatile.Read(ref resubscribed) == 0) + { + await Task.Delay(20, testContextAccessor.Current.CancellationToken); + } + + Volatile.Read(ref subscribes).Should().Be(2); + Volatile.Read(ref resubscribed).Should().Be(1); + } + + [Fact] + public async Task An_observer_that_joined_before_the_first_subscription_is_expired_once_it_is_in_place() + { + var redis = _fixture.Freeze(); + redis.Subscriber.Returns(_subscriber); + var gaps = 0; + _fixture.Inject(() => Interlocked.Increment(ref gaps)); + _options.SubscriberDueTime = TimeSpan.FromMilliseconds(200); + + Sut().MarkSubscribed(); + await WaitForSubscribeAsync(); + await Task.Delay(_delay.Multiply(3), testContextAccessor.Current.CancellationToken); + Sut().MarkSubscribed(); + await Task.Delay(_delay.Multiply(3), testContextAccessor.Current.CancellationToken); + + Volatile.Read(ref gaps).Should().Be(1, "only the observer that joined before the subscription could have missed a publication"); + } + + [Fact] + public async Task A_reconnect_during_a_subscribe_attempt_subscribes_again_on_the_new_connection() + { + var redis = _fixture.Freeze(); + redis.Subscriber.Returns(_subscriber); + _fixture.Inject(() => { }); + using var inFirstSubscribe = new ManualResetEventSlim(); + using var releaseFirst = new ManualResetEventSlim(); + var subscribes = 0; + _subscriber.When(x => x.Subscribe(_redisChannel, Arg.Any>())) + .Do(_ => + { + if (Interlocked.Increment(ref subscribes) == 1) + { + inFirstSubscribe.Set(); + releaseFirst.Wait(TimeSpan.FromSeconds(10)); + } + }); + + Sut(); + inFirstSubscribe.Wait(TimeSpan.FromSeconds(10), testContextAccessor.Current.CancellationToken).Should().BeTrue(); + redis.OnReconnected += Raise.Event(redis, EventArgs.Empty); + releaseFirst.Set(); + + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline && Volatile.Read(ref subscribes) < 2) + { + await Task.Delay(20, testContextAccessor.Current.CancellationToken); + } + + Volatile.Read(ref subscribes).Should().BeGreaterThanOrEqualTo(2, "the attempt that began before the reconnect subscribed the retired connection"); + } + + [Fact] + public async Task A_write_a_full_channel_refuses_is_reported_as_dropped() + { + var full = Channel.CreateBounded(new BoundedChannelOptions(1) { FullMode = BoundedChannelFullMode.Wait }); + full.Writer.TryWrite(new TestCacheEvent()).Should().BeTrue(); + _fixture.Inject((ChannelWriter)full.Writer); + var dropped = 0; + _fixture.Inject(() => Interlocked.Increment(ref dropped)); + var expected = _fixture.Create(); + expected.Valid = true; + + Sut(); + var action = await WaitForSubscribeAsync(); + action(_redisChannel, (RedisValue)Encoding.UTF8.GetBytes(JsonSerializer.Serialize(expected))); + + Volatile.Read(ref dropped).Should().Be(1, "the invalidation the refused write carried is gone"); + } + + [Theory] + [InlineData(true, 1)] + [InlineData(false, 0)] + public async Task A_restore_after_a_failure_reports_a_subscription_gap(bool failed, int expected) + { + var redis = _fixture.Freeze(); + redis.Subscriber.Returns(_subscriber); + var gaps = 0; + _fixture.Inject(() => Interlocked.Increment(ref gaps)); + + Sut(); + await WaitForSubscribeAsync(); + if (failed) + { + redis.OnConnectionFailed += Raise.Event(redis, EventArgs.Empty); + } + + redis.OnConnectionRestored += Raise.Event(redis, EventArgs.Empty); + redis.OnConnectionRestored += Raise.Event(redis, EventArgs.Empty); + await Task.Delay(_delay.Multiply(4), testContextAccessor.Current.CancellationToken); + + Volatile.Read(ref gaps).Should().Be(expected, "publications while the connection was down never arrived, and a second restore found no outage"); + } + + [Fact] + public async Task Only_the_restore_of_the_last_subscription_connection_down_reports_a_gap() + { + var redis = _fixture.Freeze(); + redis.Subscriber.Returns(_subscriber); + var gaps = 0; + _fixture.Inject(() => Interlocked.Increment(ref gaps)); + var first = new DnsEndPoint("node-1", 6379); + var second = new DnsEndPoint("node-2", 6379); + Sut(); + await WaitForSubscribeAsync(); + + RaiseConnectionEvent(redis, failed: true, first, ConnectionType.Subscription); + RaiseConnectionEvent(redis, failed: true, second, ConnectionType.Subscription); + RaiseConnectionEvent(redis, failed: true, first, ConnectionType.Interactive); + RaiseConnectionEvent(redis, failed: false, first, ConnectionType.Interactive); + RaiseConnectionEvent(redis, failed: false, first, ConnectionType.Subscription); + await Task.Delay(_delay.Multiply(4), testContextAccessor.Current.CancellationToken); + Volatile.Read(ref gaps).Should().Be(0, "the second node's subscription is still down"); + + RaiseConnectionEvent(redis, failed: false, second, ConnectionType.Subscription); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline && Volatile.Read(ref gaps) == 0) + { + await Task.Delay(20, testContextAccessor.Current.CancellationToken); + } + + Volatile.Read(ref gaps).Should().Be(1); + } + + [Fact] + public async Task A_failure_of_the_command_connection_alone_reports_no_gap() + { + var redis = _fixture.Freeze(); + redis.Subscriber.Returns(_subscriber); + var gaps = 0; + _fixture.Inject(() => Interlocked.Increment(ref gaps)); + var node = new DnsEndPoint("node-1", 6379); + Sut(); + await WaitForSubscribeAsync(); + + RaiseConnectionEvent(redis, failed: true, node, ConnectionType.Interactive); + RaiseConnectionEvent(redis, failed: false, node, ConnectionType.Interactive); + await Task.Delay(_delay.Multiply(4), testContextAccessor.Current.CancellationToken); + + Volatile.Read(ref gaps).Should().Be(0, "publications arrive on the subscription connection"); + } + [Fact] public void Dispose_works() { @@ -105,6 +274,24 @@ public ValueTask InitializeAsync() return ValueTask.CompletedTask; } + private static void RaiseConnectionEvent(IRedisConnector redis, bool failed, EndPoint endPoint, ConnectionType connectionType) + { + var args = (ConnectionFailedEventArgs)Activator.CreateInstance( + typeof(ConnectionFailedEventArgs), + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic, + null, + [null, null, endPoint, connectionType, ConnectionFailureType.SocketFailure, null, null], + null)!; + if (failed) + { + redis.OnConnectionFailed += NSubstitute.Raise.Event(redis, args); + } + else + { + redis.OnConnectionRestored += NSubstitute.Raise.Event(redis, args); + } + } + private RedisPubSubSubjectWriter Sut() => _sut ??= _fixture.Create>(); diff --git a/tests/UiPath.Caching.Tests/Broadcast/RedisPubSubTopicTests.cs b/tests/UiPath.Caching.Tests/Broadcast/RedisPubSubTopicTests.cs index d8a7e9c..0939dc8 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/RedisPubSubTopicTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/RedisPubSubTopicTests.cs @@ -201,6 +201,54 @@ public async Task No_exceptions_are_thrown_when_redis_fails() } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task An_observer_joining_while_the_first_subscription_goes_in_place_is_expired(bool throws) + { + var subject = new HeldSubject(); + _fixture.Inject>>(() => subject); + _options.SubscriberDueTime = TimeSpan.FromSeconds(1); + var sut = await Sut(); + var entry = new KeptEntry { Throws = throws }; + + var joining = Task.Run(() => sut.Subscribe(entry), testContextAccessor.Current.CancellationToken); + subject.Joining.Wait(TimeSpan.FromSeconds(10), testContextAccessor.Current.CancellationToken).Should().BeTrue(); + await WaitForSubscribeAsync(); + await Task.Delay(_delay.Multiply(2), testContextAccessor.Current.CancellationToken); + subject.Release.Set(); + (await joining).Dispose(); + + entry.Missed.Task.IsCompleted.Should().BeTrue("the subscription went in place after the mark but before this observer joined"); + } + + [Fact] + public async Task A_restored_connection_expires_what_the_topic_keeps() + { + var sut = await Sut(); + await WaitForSubscribeAsync(); + var entry = new KeptEntry(); + using var subscription = sut.Subscribe(entry); + + _redisConnector.OnConnectionFailed += Raise.Event(_redisConnector, EventArgs.Empty); + _redisConnector.OnConnectionRestored += Raise.Event(_redisConnector, EventArgs.Empty); + + await entry.Missed.Awaiting(m => m.Task).Should().CompleteWithinAsync(TimeSpan.FromSeconds(10), "the topic expires its entries itself, on whatever local tier holds them"); + } + + [Fact] + public async Task A_resubscribe_after_a_reconnect_expires_what_the_topic_keeps() + { + var sut = await Sut(); + await WaitForSubscribeAsync(); + var entry = new KeptEntry(); + using var subscription = sut.Subscribe(entry); + + _redisConnector.OnReconnected += Raise.Event(_redisConnector, EventArgs.Empty); + + await entry.Missed.Awaiting(m => m.Task).Should().CompleteWithinAsync(TimeSpan.FromSeconds(10), "publications between the swap and the new subscription never arrived"); + } + public ValueTask DisposeAsync() { return ValueTask.CompletedTask; @@ -275,4 +323,59 @@ private async Task> Sut(int delayMultiplier = 2) private Task WaitForSubscribeAsync() => _subscribeCalled.Task.WaitAsync(TimeSpan.FromSeconds(30), testContextAccessor.Current.CancellationToken); + + /// A subject whose next subscription waits, mid-join, until released. + private sealed class HeldSubject : IEventSubject + { + private readonly KeyedSubject _inner = new(Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + + public ManualResetEventSlim Joining { get; } = new(); + + public ManualResetEventSlim Release { get; } = new(); + + public IDisposable Subscribe(IObserver observer) + { + Joining.Set(); + Release.Wait(TimeSpan.FromSeconds(10)); + return _inner.Subscribe(observer); + } + + public void OnNext(ICacheEvent value) => _inner.OnNext(value); + + public void OnCompleted() => _inner.OnCompleted(); + + public void Invalidate(MissedEventsReason reason) => _inner.Invalidate(reason); + + public void Dispose() => _inner.Dispose(); + } + + private sealed class KeptEntry : IKeyedObserver, IMissedEventsObserver + { + public TaskCompletionSource Missed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public string Key => "order:1"; + + public bool Throws { get; init; } + + public void OnNext(ICacheEvent value) + { + } + + public void OnError(Exception error) + { + } + + public void OnCompleted() + { + } + + public void OnEventsMissed(MissedEventsReason reason) + { + Missed.TrySetResult(); + if (Throws) + { + throw new InvalidOperationException("boom"); + } + } + } } diff --git a/tests/UiPath.Caching.Tests/Broadcast/RedisStreamGapIntegrationTests.cs b/tests/UiPath.Caching.Tests/Broadcast/RedisStreamGapIntegrationTests.cs new file mode 100644 index 0000000..e36c3d7 --- /dev/null +++ b/tests/UiPath.Caching.Tests/Broadcast/RedisStreamGapIntegrationTests.cs @@ -0,0 +1,362 @@ +using System.Threading.Channels; +using Microsoft.Extensions.Logging.Abstractions; +using StackExchange.Redis; +using UiPath.Caching.Policies; +using UiPath.Caching.Telemetry; +using UiPath.Caching.Tests.Redis; + +namespace UiPath.Caching.Tests.Broadcast; + +[Collection("RedisIntegration")] +[Trait("Category", "Integration")] +public sealed class RedisStreamGapIntegrationTests(RedisContainerFixture fixture) : IAsyncLifetime +{ + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(20); + private readonly RedisKey _stream = $"it:gap:{Guid.NewGuid():N}"; + private readonly RedisValue _group = "node"; + private readonly IConnectionState _state = Substitute.For(); + private readonly CancellationTokenSource _stop = new(); + private RedisConnector? _connector; + private RedisStreamSubjectWriter? _sut; + private int _missed; + private volatile bool _connected = true; + private int _offlineChecks; + + private IDatabase Db => _connector!.Database; + + [Fact] + public async Task Entries_trimmed_during_an_outage_before_they_were_read_are_reported() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await StartReadingAsync(); + + await GoOfflineAsync(); + await AddAsync(5); + await Db.StreamTrimAsync(_stream, maxLength: 2); + Reconnect(); + + (await EventuallyAsync(() => Volatile.Read(ref _missed) == 1)).Should().BeTrue(); + } + + [Fact] + public async Task Entries_trimmed_between_two_polls_before_they_were_read_are_reported() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await StartReadingAsync(); + + // One script, so no poll lands between the adds and the trim. + await Db.ScriptEvaluateAsync( + "for i = 1, 5 do redis.call('XADD', KEYS[1], '*', 'f', 'v') end redis.call('XTRIM', KEYS[1], 'MAXLEN', 2)", + [_stream]); + + (await EventuallyAsync(() => Volatile.Read(ref _missed) == 1)).Should().BeTrue("the connection never dropped, so only a checked read can see it"); + } + + [Fact] + public async Task A_stream_trimmed_empty_is_reported_once_however_often_it_is_polled() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await StartReadingAsync(); + + await Db.ScriptEvaluateAsync( + "for i = 1, 5 do redis.call('XADD', KEYS[1], '*', 'f', 'v') end redis.call('XTRIM', KEYS[1], 'MAXLEN', 0)", + [_stream]); + (await EventuallyAsync(() => Volatile.Read(ref _missed) == 1)).Should().BeTrue(); + await Task.Delay(PollInterval * 15, TestContext.Current.CancellationToken); + Volatile.Read(ref _missed).Should().Be(1, "an empty read leaves the group where it was, so later checks find the same gap"); + + var next = await AddAsync(1); + (await EventuallyAsync(async () => await LastDeliveredAsync() == next)).Should().BeTrue(); + Volatile.Read(ref _missed).Should().Be(1, "the entry added after the trim was not lost"); + } + + [Fact] + public async Task A_new_groups_first_entries_are_not_a_gap() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + StartWriter(); + + // Reads that find nothing put the group in use before it has read an entry. + await Task.Delay(PollInterval * 5, TestContext.Current.CancellationToken); + var first = await AddAsync(1); + + (await EventuallyAsync(async () => await LastDeliveredAsync() == first)).Should().BeTrue(); + Volatile.Read(ref _missed).Should().Be(0, "a group that has not read yet counts nothing read, which no trim explains"); + } + + [Fact] + public async Task A_group_created_after_the_history_was_trimmed_away_reports_nothing_for_its_first_entries() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await Db.StreamDeleteConsumerGroupAsync(_stream, _group); + await AddAsync(5); + await Db.StreamTrimAsync(_stream, maxLength: 0); + await ConsumerGroups.CreateAsync(Db, _stream, _group, StreamPosition.NewMessages); + var first = await AddAsync(1); + + StartSubscribedOffline(); + Reconnect(); + + (await EventuallyAsync(async () => await LastDeliveredAsync() == first)).Should().BeTrue(); + Volatile.Read(ref _missed).Should().Be(0, "the entries trimmed before the group existed were never its to read"); + } + + [Fact] + public async Task Entries_trimmed_before_a_new_group_first_read_are_reported() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + StartSubscribedOffline(); + await Db.ScriptEvaluateAsync( + "for i = 1, 5 do redis.call('XADD', KEYS[1], '*', 'f', 'v') end redis.call('XTRIM', KEYS[1], 'MAXLEN', 2)", + [_stream]); + Reconnect(); + + (await EventuallyAsync(() => Volatile.Read(ref _missed) == 1)).Should().BeTrue("no poll ran before the trim, so only the count the group was created with shows it"); + } + + [Fact] + public async Task StackExchange_Redis_reads_a_new_groups_unknown_count_as_zero() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await AddAsync(3); + await Db.StreamCreateConsumerGroupAsync(_stream, "fresh", StreamPosition.NewMessages); + + Array.Find(await Db.StreamGroupInfoAsync(_stream), g => g.Name == "fresh").EntriesRead.Should().Be(0, "XINFO GROUPS replies nil, which the gap check takes as unknown"); + } + + [Fact] + public async Task Entries_kept_through_an_outage_are_replayed_without_a_report() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await StartReadingAsync(); + + await GoOfflineAsync(); + var last = await AddAsync(5); + Reconnect(); + + (await EventuallyAsync(async () => await LastDeliveredAsync() == last)).Should().BeTrue("the stream still held them"); + Volatile.Read(ref _missed).Should().Be(0); + } + + [Fact] + public async Task Trimming_entries_already_read_is_not_reported() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await StartReadingAsync(); + var last = await AddAsync(3); + (await EventuallyAsync(async () => await LastDeliveredAsync() == last)).Should().BeTrue(); + + await GoOfflineAsync(); + await Db.StreamTrimAsync(_stream, maxLength: 0); + var next = await AddAsync(1); + Reconnect(); + + (await EventuallyAsync(async () => await LastDeliveredAsync() == next)).Should().BeTrue(); + Volatile.Read(ref _missed).Should().Be(0, "the entry added after the trim is the only one not yet read"); + } + + [Fact] + public async Task A_stream_removed_while_in_use_is_reported_once() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await StartReadingAsync(); + + await Db.KeyDeleteAsync(_stream); + + (await EventuallyAsync(() => Volatile.Read(ref _missed) == 1)).Should().BeTrue(); + await Task.Delay(300, TestContext.Current.CancellationToken); + Volatile.Read(ref _missed).Should().Be(1, "the recreated group is read again before another loss counts"); + } + + [Fact] + public async Task A_group_removed_while_in_use_is_reported_once() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await StartReadingAsync(); + + await Db.StreamDeleteConsumerGroupAsync(_stream, _group); + + (await EventuallyAsync(() => Volatile.Read(ref _missed) == 1)).Should().BeTrue("the script's read fails with NOGROUP while the stream remains"); + await Task.Delay(300, TestContext.Current.CancellationToken); + Volatile.Read(ref _missed).Should().Be(1); + } + + [Fact] + public async Task A_topic_that_lost_entries_expires_what_it_keeps() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var cacheOptions = new CacheOptions { Enabled = true, AppShortName = $"it{Guid.NewGuid():N}"[..10], SourceUri = new Uri("urn:node") }; + var options = new RedisStreamsTopicOptions { Enabled = true, PollInterval = PollInterval }; + var stream = new PrefixStrategy(RedisKeyspaces.Streams, cacheOptions).GetRedisKey("orders"); + using var topic = new RedisStreamsTopic( + "orders", + _state, + _connector!, + () => new KeyedSubject(NullLogger.Instance), + Substitute.For>(), + Substitute.For(), + options, + cacheOptions, + NullLogger>.Instance, + NullTelemetryProvider.Instance, + NullRedisProfiler.Instance, + _stop.Token); + var entry = new KeptEntry(); + using var subscription = topic.Subscribe(entry); + + try + { + var id = await Db.StreamAddAsync(stream, options.FieldName, "v"); + (await EventuallyAsync(async () => Array.Find(await Db.StreamGroupInfoAsync(stream), g => g.Name == "urn:node").LastDeliveredId == id)).Should().BeTrue(); + await Db.KeyDeleteAsync(stream); + + (await Task.WhenAny(entry.Expired.Task, Task.Delay(TimeSpan.FromSeconds(10), TestContext.Current.CancellationToken))).Should().BeSameAs(entry.Expired.Task); + } + finally + { + await Db.KeyDeleteAsync(stream); + } + } + + public async ValueTask InitializeAsync() + { + if (!fixture.Enabled) + { + return; + } + + var options = Options.Create(new RedisConnectionOptions { ConnectionString = fixture.ConnectionString, EnableHangDetection = false }); + _connector = new RedisConnector( + NullTelemetryProvider.Instance, + new RedisConfigurationOptionsProvider(NullLoggerFactory.Instance, options), + new ConnectionMultiplexerFactory(options, NullRedisProfiler.Instance), + options); + await _connector.ConnectAsync(TestContext.Current.CancellationToken); + await ConsumerGroups.CreateAsync(Db, _stream, _group, StreamPosition.NewMessages); + _state.IsConnected.Returns(_ => + { + if (_connected) + { + return true; + } + + Interlocked.Increment(ref _offlineChecks); + return false; + }); + } + + public async ValueTask DisposeAsync() + { + await _stop.CancelAsync(); + _sut?.Dispose(); + if (_connector is not null) + { + await Db.KeyDeleteAsync(_stream); + _connector.Dispose(); + } + + _stop.Dispose(); + } + + private static Task EventuallyAsync(Func condition) => EventuallyAsync(() => Task.FromResult(condition())); + + private static async Task EventuallyAsync(Func> condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline) + { + if (await condition()) + { + return true; + } + + await Task.Delay(20, TestContext.Current.CancellationToken); + } + + return false; + } + + private async Task StartReadingAsync() + { + StartWriter(); + var first = await AddAsync(1); + (await EventuallyAsync(async () => await LastDeliveredAsync() == first)).Should().BeTrue("the group is in use once it has read"); + } + + // Subscribed before the writer could read, as a topic does, so its first read is checked. + private void StartSubscribedOffline() + { + _connected = false; + StartWriter(); + _sut!.MarkSubscribed(); + } + + private void StartWriter() + { + var context = new RedisStreamContext(_stream, "f", _group, _group, new Uri("urn:node"), 100, PollInterval, false, false); + _sut = new RedisStreamSubjectWriter( + context, + _state, + _connector!, + Channel.CreateUnbounded().Writer, + Substitute.For>(), + NullLogger.Instance, + NullTelemetryProvider.Instance, + NullRedisProfiler.Instance, + new TimedFetchWaiter(PollInterval), + () => Interlocked.Increment(ref _missed), + _stop.Token); + } + + private async Task AddAsync(int count) + { + RedisValue id = default; + for (var i = 0; i < count; i++) + { + id = await Db.StreamAddAsync(_stream, "f", "v"); + } + + return id; + } + + private async Task LastDeliveredAsync() + { + var groups = await Db.StreamGroupInfoAsync(_stream); + return Array.Find(groups, g => g.Name == _group).LastDeliveredId; + } + + // Waits for the reader to see the outage itself: a read already past its connection check would otherwise still run. + private async Task GoOfflineAsync() + { + var seen = Volatile.Read(ref _offlineChecks); + _connected = false; + (await EventuallyAsync(() => Volatile.Read(ref _offlineChecks) > seen)).Should().BeTrue("the reader parks before the outage's entries are added"); + } + + private void Reconnect() + { + _connected = true; + _state.OnConnectionRestored += Raise.Event(_state, EventArgs.Empty); + } + + private sealed class KeptEntry : IKeyedObserver, IMissedEventsObserver + { + public TaskCompletionSource Expired { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public string Key => "order:1"; + + public void OnNext(ICacheEvent value) + { + } + + public void OnEventsMissed(MissedEventsReason reason) => Expired.TrySetResult(); + + public void OnError(Exception error) + { + } + + public void OnCompleted() + { + } + } +} diff --git a/tests/UiPath.Caching.Tests/Broadcast/RedisStreamSubjectWriterTests.cs b/tests/UiPath.Caching.Tests/Broadcast/RedisStreamSubjectWriterTests.cs index d291480..e82a1b0 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/RedisStreamSubjectWriterTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/RedisStreamSubjectWriterTests.cs @@ -2,6 +2,7 @@ using System.Threading.Channels; using Microsoft.Extensions.Logging; using NSubstitute.ExceptionExtensions; +using NSubstitute.Extensions; using NSubstitute.ReceivedExtensions; using StackExchange.Redis; using UiPath.Caching.Telemetry; @@ -229,6 +230,289 @@ public async Task NOGROUP_error_triggers_StreamCreateConsumerGroup() await _database.ReceivedWithAnyArgs().StreamCreateConsumerGroupAsync(_context.Topic, _context.ConsumerGroup, Arg.Any()); } + [Fact] + public async Task A_group_lost_after_it_was_read_reports_missed_messages_once() + { + ScriptsUnavailable(); + var reads = 0; + _database.StreamReadGroupAsync(_context.Topic, _context.ConsumerGroup, _context.ConsumerName, ">", _context.PollBatchSize) + .ReturnsForAnyArgs(_ => Interlocked.Increment(ref reads) == 1 + ? Task.FromResult(Array.Empty()) + : Task.FromException(new RedisException("NOGROUP No such key or consumer group"))); + _database.StreamCreateConsumerGroupAsync(_context.Topic, _context.ConsumerGroup, Arg.Any()).ReturnsForAnyArgs(false); + var missed = 0; + + using var sut = CreateSut(Channel.CreateUnbounded().Writer, _logger, onMessagesMissed: () => Interlocked.Increment(ref missed)); + + (await WaitUntil(() => Volatile.Read(ref reads) >= 4)).Should().BeTrue(); + Volatile.Read(ref missed).Should().Be(1, "the group is not read again in between"); + } + + [Fact] + public async Task A_connection_seen_down_is_checked_for_a_gap_before_the_next_read() + { + var connected = 1; + var offlineChecks = 0; + var readsAfterOutage = 0; + var gapChecks = 0; + var checkedFirst = false; + _database.StreamReadGroupAsync(_context.Topic, _context.ConsumerGroup, _context.ConsumerName, ">", _context.PollBatchSize) + .ReturnsForAnyArgs(_ => + { + if (Volatile.Read(ref offlineChecks) > 0 && Interlocked.Increment(ref readsAfterOutage) == 1) + { + Volatile.Write(ref checkedFirst, Volatile.Read(ref gapChecks) > 0); + } + + return Task.FromResult(Array.Empty()); + }); + _database.Configure().StreamInfoAsync(_context.Topic).ReturnsForAnyArgs(_ => + { + Interlocked.Increment(ref gapChecks); + return Task.FromResult(default(StreamInfo)); + }); + _database.Configure().StreamGroupInfoAsync(_context.Topic).ReturnsForAnyArgs(Task.FromResult(Array.Empty())); + ScriptsUnavailable(); + var connectionState = _fixture.Create(); + connectionState.IsConnected.Returns(_ => + { + if (Volatile.Read(ref connected) == 1) + { + return true; + } + + Interlocked.Increment(ref offlineChecks); + return false; + }); + var redis = _fixture.Create(); + redis.Database.Returns(_database); + + using var sut = new RedisStreamSubjectWriter( + _context, + connectionState, + redis, + Channel.CreateUnbounded().Writer, + _formatter, + _logger, + _fixture.Create(), + _fixture.Create(), + new TimedFetchWaiter(_pollInterval), + () => { }, + _cancellationTokenSource.Token); + + (await WaitUntil(() => _database.ReceivedCalls().Any(c => c.GetMethodInfo().Name == nameof(IDatabase.StreamReadGroupAsync)))).Should().BeTrue(); + Volatile.Write(ref connected, 0); + (await WaitUntil(() => Volatile.Read(ref offlineChecks) > 0)).Should().BeTrue(); + Volatile.Write(ref connected, 1); + + (await WaitUntil(() => Volatile.Read(ref readsAfterOutage) > 0)).Should().BeTrue(); + Volatile.Read(ref checkedFirst).Should().BeTrue("the restored event may not have fired yet"); + } + + [Fact] + public async Task A_recovery_raised_during_a_gap_check_is_checked_before_the_next_read() + { + var gapChecks = 0; + var readBetweenChecks = false; + var connectionState = _fixture.Create(); + connectionState.IsConnected.Returns(true); + _database.StreamReadGroupAsync(_context.Topic, _context.ConsumerGroup, _context.ConsumerName, ">", _context.PollBatchSize) + .ReturnsForAnyArgs(_ => + { + if (Volatile.Read(ref gapChecks) == 1) + { + Volatile.Write(ref readBetweenChecks, true); + } + + return Task.FromResult(Array.Empty()); + }); + _database.Configure().StreamInfoAsync(_context.Topic).ReturnsForAnyArgs(_ => + { + if (Interlocked.Increment(ref gapChecks) == 1) + { + connectionState.OnConnectionRestored += Raise.Event(connectionState, EventArgs.Empty); + } + + return Task.FromResult(default(StreamInfo)); + }); + _database.Configure().StreamGroupInfoAsync(_context.Topic).ReturnsForAnyArgs(Task.FromResult(Array.Empty())); + ScriptsUnavailable(); + var redis = _fixture.Create(); + redis.Database.Returns(_database); + + using var sut = new RedisStreamSubjectWriter( + _context, + connectionState, + redis, + Channel.CreateUnbounded().Writer, + _formatter, + _logger, + _fixture.Create(), + _fixture.Create(), + new TimedFetchWaiter(_pollInterval), + () => { }, + _cancellationTokenSource.Token); + + (await WaitUntil(() => _database.ReceivedCalls().Any(c => c.GetMethodInfo().Name == nameof(IDatabase.StreamReadGroupAsync)))).Should().BeTrue(); + connectionState.OnConnectionRestored += Raise.Event(connectionState, EventArgs.Empty); + + (await WaitUntil(() => Volatile.Read(ref gapChecks) >= 2)).Should().BeTrue("the second recovery came after the first check began"); + Volatile.Read(ref readBetweenChecks).Should().BeFalse("a read in between moves the group past the entries the second check compares"); + } + + [Theory] + [InlineData("NOPERM this user has no permissions to run the 'xinfo|stream' command")] + [InlineData("ERR unknown command 'XINFO'")] + public async Task A_denied_gap_check_reports_a_loss_and_keeps_reading(string error) + { + var connectionState = _fixture.Create(); + connectionState.IsConnected.Returns(true); + var reads = 0; + _database.StreamReadGroupAsync(_context.Topic, _context.ConsumerGroup, _context.ConsumerName, ">", _context.PollBatchSize) + .ReturnsForAnyArgs(_ => + { + Interlocked.Increment(ref reads); + return Task.FromResult(Array.Empty()); + }); + ScriptsUnavailable(); + _database.Configure().StreamInfoAsync(_context.Topic).ReturnsForAnyArgs(Task.FromException(error.StartsWith("ERR", StringComparison.Ordinal) ? UnknownCommandError(error) : new RedisServerException(RedisErrorKind.None, CommandFlags.None, error))); + var redis = _fixture.Create(); + redis.Database.Returns(_database); + var missed = 0; + + using var sut = new RedisStreamSubjectWriter( + _context, + connectionState, + redis, + Channel.CreateUnbounded().Writer, + _formatter, + _logger, + _fixture.Create(), + _fixture.Create(), + new TimedFetchWaiter(_pollInterval), + () => Interlocked.Increment(ref missed), + _cancellationTokenSource.Token); + + (await WaitUntil(() => Volatile.Read(ref reads) > 0)).Should().BeTrue(); + connectionState.OnConnectionRestored += Raise.Event(connectionState, EventArgs.Empty); + (await WaitUntil(() => Volatile.Read(ref missed) == 1)).Should().BeTrue("a loss cannot be ruled out without XINFO"); + var readsAfterLoss = Volatile.Read(ref reads); + + (await WaitUntil(() => Volatile.Read(ref reads) > readsAfterLoss + 2)).Should().BeTrue("the denied check must not stop consumption"); + Volatile.Read(ref missed).Should().Be(1); + } + + [Fact] + public async Task Scripts_refused_by_the_client_fall_back_to_checking_and_reading_apart() + { + var connectionState = _fixture.Create(); + connectionState.IsConnected.Returns(true); + _database.StreamReadGroupAsync(_context.Topic, _context.ConsumerGroup, _context.ConsumerName, ">", _context.PollBatchSize) + .ReturnsForAnyArgs(Task.FromResult(Array.Empty())); + _database.Configure().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsForAnyArgs(Task.FromException(new RedisCommandException("This operation has been disabled in the command-map and cannot be used: EVALSHA"))); + var infos = 0; + _database.Configure().StreamInfoAsync(_context.Topic).ReturnsForAnyArgs(_ => + { + Interlocked.Increment(ref infos); + return Task.FromResult(default(StreamInfo)); + }); + _database.Configure().StreamGroupInfoAsync(_context.Topic).ReturnsForAnyArgs(Task.FromResult(Array.Empty())); + var redis = _fixture.Create(); + redis.Database.Returns(_database); + + using var sut = new RedisStreamSubjectWriter( + _context, + connectionState, + redis, + Channel.CreateUnbounded().Writer, + _formatter, + _logger, + _fixture.Create(), + _fixture.Create(), + new TimedFetchWaiter(_pollInterval), + () => { }, + _cancellationTokenSource.Token); + + (await WaitUntil(() => _database.ReceivedCalls().Any(c => c.GetMethodInfo().Name == nameof(IDatabase.StreamReadGroupAsync)))).Should().BeTrue(); + connectionState.OnConnectionRestored += Raise.Event(connectionState, EventArgs.Empty); + + (await WaitUntil(() => Volatile.Read(ref infos) > 0)).Should().BeTrue("a command map that disables scripts must not stop the gap check"); + } + + [Fact] + public async Task A_read_after_a_partial_batch_is_checked_for_a_gap() + { + _formatter.Decode(Arg.Any>()).Returns(_ => new TestCacheEvent { Valid = true, Source = new Uri("urn:other") }); + SetupSingleBatch([new StreamEntry("1-0", [new NameValueEntry(_fieldName, "a")])]); + var scripts = 0; + _database.Configure().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsForAnyArgs(_ => + { + Interlocked.Increment(ref scripts); + return Task.FromResult(UntrimmedReply()); + }); + + using var sut = CreateSut(Channel.CreateUnbounded().Writer, _logger); + + (await WaitUntil(() => Volatile.Read(ref scripts) > 0)).Should().BeTrue("a trim can reach unread entries between any two polls"); + } + + [Fact] + public async Task A_pending_gap_check_reads_in_the_same_script() + { + var connectionState = _fixture.Create(); + connectionState.IsConnected.Returns(true); + _database.StreamReadGroupAsync(_context.Topic, _context.ConsumerGroup, _context.ConsumerName, ">", _context.PollBatchSize) + .ReturnsForAnyArgs(Task.FromResult(Array.Empty())); + var scripts = 0; + _database.Configure().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsForAnyArgs(_ => + { + Interlocked.Increment(ref scripts); + return Task.FromResult(UntrimmedReply()); + }); + var redis = _fixture.Create(); + redis.Database.Returns(_database); + + using var sut = new RedisStreamSubjectWriter( + _context, + connectionState, + redis, + Channel.CreateUnbounded().Writer, + _formatter, + _logger, + _fixture.Create(), + _fixture.Create(), + new TimedFetchWaiter(_pollInterval), + () => { }, + _cancellationTokenSource.Token); + + (await WaitUntil(() => _database.ReceivedCalls().Any(c => c.GetMethodInfo().Name == nameof(IDatabase.StreamReadGroupAsync)))).Should().BeTrue(); + connectionState.OnConnectionRestored += Raise.Event(connectionState, EventArgs.Empty); + + (await WaitUntil(() => Volatile.Read(ref scripts) > 0)).Should().BeTrue(); + _database.ReceivedCalls().Should().NotContain(c => c.GetMethodInfo().Name == nameof(IDatabase.StreamInfoAsync), "a trim must not land between a check and the read it guards"); + } + + [Fact] + public async Task A_group_missing_before_the_first_read_reports_nothing() + { + var reads = 0; + _database.StreamReadGroupAsync(_context.Topic, _context.ConsumerGroup, _context.ConsumerName, ">", _context.PollBatchSize) + .ReturnsForAnyArgs(_ => + { + Interlocked.Increment(ref reads); + return Task.FromException(new RedisException("NOGROUP No such key or consumer group")); + }); + var missed = 0; + + using var sut = CreateSut(Channel.CreateUnbounded().Writer, _logger, onMessagesMissed: () => Interlocked.Increment(ref missed)); + + (await WaitUntil(() => Volatile.Read(ref reads) >= 3)).Should().BeTrue(); + Volatile.Read(ref missed).Should().Be(0, "nothing was read through the group yet"); + } + [Fact] public async Task NOGROUP_recovery_swallows_BUSYGROUP_when_group_already_exists() { @@ -348,6 +632,7 @@ public async Task Unknown_command_quarantine_is_lifted_when_the_connection_recon _fixture.Create(), _fixture.Create(), new TimedFetchWaiter(_pollInterval), + () => { }, _cancellationTokenSource.Token); await loggedCritical.Task.WaitAsync(WaitTimeout, TestContext.Current.CancellationToken); @@ -589,7 +874,7 @@ public ValueTask InitializeAsync() _consumerName = _fixture.Create(); _consumerGroup = _fixture.Create(); _sourceUri = new Uri("urn:" + _fixture.Create()); - _pollBatchSize = _fixture.Create(); + _pollBatchSize = 100; _pollInterval = DefaultPollInterval; _context = new RedisStreamContext(_topic, _fieldName, _consumerName, _consumerGroup, _sourceUri, _pollBatchSize, _pollInterval, false, true); _fixture.Inject(_context); @@ -607,6 +892,10 @@ public ValueTask InitializeAsync() _fixture.Inject(_formatter); _fixture.Inject(new TimedFetchWaiter(_pollInterval)); + _database.Configure().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsForAnyArgs(_ => Task.FromResult(UntrimmedReply())); + _database.Configure().StreamInfoAsync(Arg.Any()).ReturnsForAnyArgs(Task.FromResult(default(StreamInfo))); + _database.Configure().StreamGroupInfoAsync(Arg.Any()).ReturnsForAnyArgs(Task.FromResult(Array.Empty())); return ValueTask.CompletedTask; } @@ -628,6 +917,13 @@ private static async Task WaitUntil(Func condition) private static RedisServerException UnknownCommandError(string message) => new(RedisErrorKind.UnknownCommand, CommandFlags.None, message); + private static RedisResult UntrimmedReply() => RedisResult.Create( + [ + RedisResult.Create(new RedisValue[] { "length", 0, "last-generated-id", "1-0", "entries-added", 1 }), + RedisResult.Create([RedisResult.Create(new RedisValue[] { "name", "group", "last-delivered-id", "1-0", "entries-read", 1 })]), + RedisResult.Create(RedisValue.Null), + ]); + private void SetupSingleBatch(StreamEntry[] entries) { var emitted = 0; @@ -635,7 +931,11 @@ private void SetupSingleBatch(StreamEntry[] entries) .ReturnsForAnyArgs(_ => ++emitted == 1 ? entries : []); } - private RedisStreamSubjectWriter CreateSut(ChannelWriter writer, ILogger logger, ICachingTelemetryProvider? telemetry = null) + private void ScriptsUnavailable() => + _database.Configure().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsForAnyArgs(Task.FromException(UnknownCommandError("ERR unknown command 'EVALSHA'"))); + + private RedisStreamSubjectWriter CreateSut(ChannelWriter writer, ILogger logger, ICachingTelemetryProvider? telemetry = null, Action? onMessagesMissed = null) { var connectionState = _fixture.Create(); connectionState.IsConnected.Returns(true); @@ -651,6 +951,7 @@ private RedisStreamSubjectWriter CreateSut(ChannelWriter(), _fixture.Create(), new TimedFetchWaiter(_pollInterval), + onMessagesMissed ?? (() => { }), _cancellationTokenSource.Token); } diff --git a/tests/UiPath.Caching.Tests/Broadcast/RedisStreamsTopicTests.cs b/tests/UiPath.Caching.Tests/Broadcast/RedisStreamsTopicTests.cs index cc61424..f8305ab 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/RedisStreamsTopicTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/RedisStreamsTopicTests.cs @@ -1,4 +1,5 @@ using NSubstitute.ExceptionExtensions; +using NSubstitute.Extensions; using StackExchange.Redis; using UiPath.Caching.Policies; @@ -244,6 +245,95 @@ public async Task Publish_returns_true_when_doorbell_publish_throws_synchronousl ok.Should().BeTrue(); } + [Fact] + public async Task A_stream_removed_while_in_use_invalidates_the_subject() + { + _database.StreamReadGroupAsync((RedisKey)"stream", "group", "consumer", ">", 1) + .ReturnsForAnyArgs( + Task.FromResult(Array.Empty()), + Task.FromException(new RedisException("NOGROUP No such key or consumer group")), + Task.FromResult(Array.Empty())); + _redisStreamsTopicOptions.PollInterval = TimeSpan.FromMilliseconds(20); + _redisStreamsTopicOptions.NotifyEnabled = false; + _ = Sut; + + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline && !_subject.ReceivedCalls().Any(c => c.GetMethodInfo().Name == nameof(IEventSubject.Invalidate))) + { + await Task.Delay(20, testContextAccessor.Current.CancellationToken); + } + + _subject.Received(1).Invalidate(MissedEventsReason.Lost); + } + + [Fact] + public async Task A_stream_removed_after_a_subscribe_but_before_the_first_read_invalidates_the_subject() + { + var subscribed = false; + var readsSinceSubscribed = 0; + _database.StreamReadGroupAsync((RedisKey)"stream", "group", "consumer", ">", 1) + .ReturnsForAnyArgs(_ => !Volatile.Read(ref subscribed) || Interlocked.Increment(ref readsSinceSubscribed) == 1 + ? Task.FromException(new RedisException("NOGROUP No such key or consumer group")) + : Task.FromResult(Array.Empty())); + _redisStreamsTopicOptions.PollInterval = TimeSpan.FromMilliseconds(20); + _redisStreamsTopicOptions.NotifyEnabled = false; + + using var subscription = Sut.Subscribe(_observer); + Volatile.Write(ref subscribed, true); + + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline && Volatile.Read(ref readsSinceSubscribed) < 3) + { + await Task.Delay(20, testContextAccessor.Current.CancellationToken); + } + + _subject.Received(1).Invalidate(MissedEventsReason.Lost); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task An_observer_joining_while_a_loss_is_reported_is_told_itself(bool throws) + { + var lose = false; + var lost = 0; + _database.StreamReadGroupAsync((RedisKey)"stream", "group", "consumer", ">", 1) + .ReturnsForAnyArgs(_ => Volatile.Read(ref lose) && Interlocked.Increment(ref lost) == 1 + ? Task.FromException(new RedisException("NOGROUP No such key or consumer group")) + : Task.FromResult(Array.Empty())); + _redisStreamsTopicOptions.PollInterval = TimeSpan.FromMilliseconds(20); + _redisStreamsTopicOptions.NotifyEnabled = false; + using var joining = new ManualResetEventSlim(); + using var release = new ManualResetEventSlim(); + _subject.Subscribe(Arg.Any>()).Returns(_ => + { + joining.Set(); + release.Wait(TimeSpan.FromSeconds(10)); + return Substitute.For(); + }); + var entry = Substitute.For, IMissedEventsObserver>(); + if (throws) + { + ((IMissedEventsObserver)entry).When(e => e.OnEventsMissed(Arg.Any())).Do(_ => throw new InvalidOperationException("boom")); + } + + _ = Sut; + + var subscribing = Task.Run(() => Sut.Subscribe(entry), testContextAccessor.Current.CancellationToken); + joining.Wait(TimeSpan.FromSeconds(10), testContextAccessor.Current.CancellationToken).Should().BeTrue(); + Volatile.Write(ref lose, true); + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10); + while (DateTime.UtcNow < deadline && !_subject.ReceivedCalls().Any(c => c.GetMethodInfo().Name == nameof(IEventSubject.Invalidate))) + { + await Task.Delay(20, testContextAccessor.Current.CancellationToken); + } + + release.Set(); + (await subscribing).Dispose(); + + ((IMissedEventsObserver)entry).Received(1).OnEventsMissed(MissedEventsReason.Lost); + } + public ValueTask DisposeAsync() { _sut?.Dispose(); @@ -254,6 +344,13 @@ public ValueTask DisposeAsync() public ValueTask InitializeAsync() { _database = _fixture.Freeze(); + + // Reads go apart from the gap check, so these tests drive them through StreamReadGroupAsync. + _database.Configure().ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsForAnyArgs(Task.FromException(new RedisCommandException("scripts are disabled"))); + _database.Configure().StreamInfo(Arg.Any()).ReturnsForAnyArgs(default(StreamInfo)); + _database.Configure().StreamInfoAsync(Arg.Any()).ReturnsForAnyArgs(Task.FromResult(default(StreamInfo))); + _database.Configure().StreamGroupInfoAsync(Arg.Any()).ReturnsForAnyArgs(Task.FromResult(Array.Empty())); _subject = _fixture.Freeze>(); _observer = _fixture.Freeze>(); _connectionState = _fixture.Freeze(); diff --git a/tests/UiPath.Caching.Tests/Broadcast/StreamIdsTests.cs b/tests/UiPath.Caching.Tests/Broadcast/StreamIdsTests.cs new file mode 100644 index 0000000..4e4374b --- /dev/null +++ b/tests/UiPath.Caching.Tests/Broadcast/StreamIdsTests.cs @@ -0,0 +1,100 @@ +using StackExchange.Redis; + +namespace UiPath.Caching.Tests.Broadcast; + +public class StreamIdsTests +{ + [Fact] + public void Entries_reads_the_ids_and_fields_of_an_XREADGROUP_reply() + { + var reply = RedisResult.Create( + [ + RedisResult.Create( + [ + RedisResult.Create((RedisValue)"stream"), + RedisResult.Create( + [ + RedisResult.Create([RedisResult.Create((RedisValue)"1-0"), RedisResult.Create(new RedisValue[] { "f", "a" })]), + RedisResult.Create([RedisResult.Create((RedisValue)"2-0"), RedisResult.Create(new RedisValue[] { "f", "b", "g", "c" })]), + ]), + ]), + ]); + + var entries = StreamIds.Entries(reply); + + entries.Select(e => (string?)e.Id).Should().Equal("1-0", "2-0"); + entries[0]["f"].Should().Be((RedisValue)"a"); + entries[1]["g"].Should().Be((RedisValue)"c"); + } + + [Fact] + public void Entries_of_a_nil_reply_are_none() => + StreamIds.Entries(RedisResult.Create(RedisValue.Null)).Should().BeEmpty(); + + [Theory] + [InlineData("3-0", "0-0", 6L, 3L, true)] + [InlineData("3-0", "0-0", 4L, 2L, false)] + [InlineData("1-0", "0-0", 6L, 3L, false)] + [InlineData("3-0", "0-0", 2L, null, true)] + [InlineData("1-0", "0-0", 6L, null, false)] + [InlineData("3-0", "2-5", 4L, null, true)] + [InlineData("3-0", null, 4L, 2L, true)] + [InlineData("1-0", null, 4L, 2L, false)] + public void Gap_reads_the_XINFO_replies(string firstEntry, string? maxDeleted, long entriesAdded, long? entriesRead, bool removed) + { + ReadCount? known = null; + + (StreamIds.Gap(Stream(2, "9-0", entriesAdded, firstEntry, maxDeleted), Groups("2-0", entriesRead), "node", ref known) is not null) + .Should().Be(removed); + } + + [Fact] + public void A_count_Redis_cannot_tell_is_every_entry_added_at_the_newest_entry_and_kept_until_the_group_reads_on() + { + ReadCount? known = null; + + // A group created at $ on a stream whose history was trimmed away. + StreamIds.Gap(Stream(0, "9-0", 6, null, "0-0"), Groups("9-0", null), "node", ref known).Should().BeNull(); + known.Should().Be(new ReadCount("9-0", 6)); + StreamIds.Gap(Stream(1, "10-0", 7, "10-0", "0-0"), Groups("9-0", null), "node", ref known).Should().BeNull("the one entry added since is still there"); + StreamIds.Gap(Stream(2, "14-0", 12, "13-0", "0-0"), Groups("9-0", null), "node", ref known).Should().NotBeNull("three of the five added since were trimmed"); + } + + [Fact] + public void A_count_Redis_cannot_tell_after_the_group_read_on_is_taken_as_a_trim() + { + ReadCount? known = new ReadCount("2-0", 1); + + StreamIds.Gap(Stream(2, "9-0", 10, "7-0", "0-0"), Groups("5-0", null), "node", ref known).Should().NotBeNull(); + known.Should().BeNull(); + } + + private static RedisResult Stream(long length, string lastGenerated, long entriesAdded, string? firstEntry, string? maxDeleted) + { + var fields = new List + { + RedisResult.Create((RedisValue)"length"), RedisResult.Create((RedisValue)length), + RedisResult.Create((RedisValue)"last-generated-id"), RedisResult.Create((RedisValue)lastGenerated), + RedisResult.Create((RedisValue)"entries-added"), RedisResult.Create((RedisValue)entriesAdded), + }; + if (firstEntry is not null) + { + fields.Add(RedisResult.Create((RedisValue)"first-entry")); + fields.Add(RedisResult.Create([RedisResult.Create((RedisValue)firstEntry)])); + } + + if (maxDeleted is not null) + { + fields.Add(RedisResult.Create((RedisValue)"max-deleted-entry-id")); + fields.Add(RedisResult.Create((RedisValue)maxDeleted)); + } + + return RedisResult.Create([.. fields]); + } + + private static RedisResult Groups(string lastDelivered, long? entriesRead) => RedisResult.Create( + [ + RedisResult.Create(new RedisValue[] { "name", "other", "last-delivered-id", "0-0" }), + RedisResult.Create(new RedisValue[] { "name", "node", "last-delivered-id", lastDelivered, "entries-read", entriesRead is { } read ? read : RedisValue.Null }), + ]); +} diff --git a/tests/UiPath.Caching.Tests/CacheEventPublisherTests.cs b/tests/UiPath.Caching.Tests/CacheEventPublisherTests.cs new file mode 100644 index 0000000..33cdff8 --- /dev/null +++ b/tests/UiPath.Caching.Tests/CacheEventPublisherTests.cs @@ -0,0 +1,37 @@ +using Microsoft.Extensions.Logging.Abstractions; + +namespace UiPath.Caching.Tests; + +public class CacheEventPublisherTests +{ + public static TheoryData Raises => new() + { + { nameof(CacheEventPublisher.MetadataUpdatedAsync), KnownEventTypes.CacheRefreshed }, + { nameof(CacheEventPublisher.CacheSetAsync), KnownEventTypes.CacheSet }, + { nameof(CacheEventPublisher.CacheRefreshedAsync), KnownEventTypes.CacheRefreshed }, + { nameof(CacheEventPublisher.CacheRemovedAsync), KnownEventTypes.CacheRemoved }, + }; + + [Theory] + [MemberData(nameof(Raises))] + public async Task Each_notification_raises_its_own_event_type(string method, string eventType) + { + var factory = Substitute.For(); + var topics = Substitute.For(); + topics.Create(Arg.Any()).PublishAsync(Arg.Any(), Arg.Any()).Returns(true); + var options = Substitute.For(); + options.CacheKey.Returns(new CacheKey("user:42")); + var sut = new CacheEventPublisher("cache", topics, factory, NullLogger.Instance); + + var raised = method switch + { + nameof(CacheEventPublisher.MetadataUpdatedAsync) => await sut.MetadataUpdatedAsync(options), + nameof(CacheEventPublisher.CacheSetAsync) => await sut.CacheSetAsync(options), + nameof(CacheEventPublisher.CacheRefreshedAsync) => await sut.CacheRefreshedAsync(options), + _ => await sut.CacheRemovedAsync(options), + }; + + raised.Should().BeTrue(); + factory.Received(1).Create("cache", eventType, Arg.Any()); + } +} diff --git a/tests/UiPath.Caching.Tests/Fakes/InMemoryMultilayer.cs b/tests/UiPath.Caching.Tests/Fakes/InMemoryMultilayer.cs index a94d248..2feb9f8 100644 --- a/tests/UiPath.Caching.Tests/Fakes/InMemoryMultilayer.cs +++ b/tests/UiPath.Caching.Tests/Fakes/InMemoryMultilayer.cs @@ -8,16 +8,17 @@ namespace UiPath.Caching.Tests.Fakes; /// Real multilayer caches over a real MemoryCache and the null inner tier, as the in-memory provider builds them. internal static class InMemoryMultilayer { - public static MultilayerCache Cache(InMemoryCacheOptions? options = null, ICache? inner = null, bool connectionMonitor = false) + public static MultilayerCache Cache(InMemoryCacheOptions? options = null, ICache? inner = null, bool connectionMonitor = false, TimeProvider? clock = null, ITopicFactory? topics = null, IChangeTokenFactory? tokens = null) { + clock ??= TimeProvider.System; options ??= new InMemoryCacheOptions(); var cacheOptions = new CacheOptions { AppShortName = "test", ConnectionMonitorEnabled = connectionMonitor }; return new MultilayerCache( KnownCacheProviderNames.InMemory, inner ?? NullCache.Instance, - new MemoryCacheFactory(TimeProvider.System, NullLoggerFactory.Instance), - NullChangeTokenFactory.Instance, - NullTopicFactory.Instance, + new MemoryCacheFactory(clock, NullLoggerFactory.Instance), + tokens ?? NullChangeTokenFactory.Instance, + topics ?? NullTopicFactory.Instance, NullCacheEventFactory.Instance, NullTelemetryProvider.Instance, options, @@ -26,7 +27,7 @@ public static MultilayerCache Cache(InMemoryCacheOptions? options = null, ICache localLock: new AsyncKeyedLocalLock(Options.Create(cacheOptions)), distributedLock: NullDistributedLock.Instance, policyFactory: NullCachePolicyFactory.Instance, - clock: TimeProvider.System, + clock: clock, logger: NullLogger.Instance); } diff --git a/tests/UiPath.Caching.Tests/InFlightTests.cs b/tests/UiPath.Caching.Tests/InFlightTests.cs new file mode 100644 index 0000000..e32e0ae --- /dev/null +++ b/tests/UiPath.Caching.Tests/InFlightTests.cs @@ -0,0 +1,234 @@ +namespace UiPath.Caching.Tests; + +public class InFlightTests +{ + private const string AbandonedFailure = "work failed after every caller left"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Concurrent_callers_for_one_key_share_one_run() + { + var flights = new InFlight(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var runs = 0; + + var callers = Enumerable.Range(0, 10) + .Select(_ => flights.RunAsync("k", release, (r, _) => { Interlocked.Increment(ref runs); return new ValueTask(r.Task); }, Ct).AsTask()) + .ToArray(); + release.SetResult(42); + + (await Task.WhenAll(callers)).Should().AllBeEquivalentTo(42); + runs.Should().Be(1); + } + + [Fact] + public async Task Different_keys_run_separately() + { + var flights = new InFlight(); + + var a = await flights.RunAsync("a", "A", static (s, _) => new ValueTask(s), Ct); + var b = await flights.RunAsync("b", "B", static (s, _) => new ValueTask(s), Ct); + + (a, b).Should().Be(("A", "B")); + } + + [Fact] + public async Task A_caller_cancelled_before_it_arrives_starts_no_work() + { + var flights = new InFlight(); + using var cancelled = new CancellationTokenSource(); + await cancelled.CancelAsync(); + var runs = 0; + + var call = () => flights.RunAsync("k", 0, (_, _) => new ValueTask(Interlocked.Increment(ref runs)), cancelled.Token).AsTask(); + + await call.Should().ThrowAsync(); + runs.Should().Be(0); + flights.Count.Should().Be(0); + } + + + [Fact] + public async Task Work_every_caller_left_fails_without_an_unobserved_exception() + { + var unobserved = 0; + void OnUnobserved(object? sender, UnobservedTaskExceptionEventArgs e) + { + if (e.Exception.InnerExceptions.Any(x => x.Message == AbandonedFailure)) + { + Interlocked.Increment(ref unobserved); + } + } + + TaskScheduler.UnobservedTaskException += OnUnobserved; + try + { + await AbandonAFailingRunAsync(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + Volatile.Read(ref unobserved).Should().Be(0); + } + finally + { + TaskScheduler.UnobservedTaskException -= OnUnobserved; + } + } + + [Fact] + public async Task Work_every_caller_left_cannot_commit() + { + var flights = new InFlight(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var committed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var leaving = new CancellationTokenSource(); + async ValueTask IgnoreCancellation(TaskCompletionSource gate, IInFlightRun run) + { + await gate.Task; + committed.SetResult(run.TryCommit(0, static _ => { })); + return 0; + } + + var abandoned = flights.RunAsync("k", release, IgnoreCancellation, leaving.Token).AsTask(); + await leaving.CancelAsync(); + await abandoned.Invoking(t => t).Should().ThrowAsync(); + release.SetResult(); + + (await committed.Task.WaitAsync(TimeSpan.FromSeconds(5), Ct)).Should().BeFalse(); + } + + [Fact] + public async Task A_finished_run_is_not_reused() + { + var flights = new InFlight(); + var runs = 0; + + await flights.RunAsync("k", 0, (_, _) => new ValueTask(Interlocked.Increment(ref runs)), Ct); + var second = await flights.RunAsync("k", 0, (_, _) => new ValueTask(Interlocked.Increment(ref runs)), Ct); + + second.Should().Be(2); + } + + [Fact] + public async Task A_failure_reaches_every_caller() + { + var flights = new InFlight(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var callers = Enumerable.Range(0, 3) + .Select(_ => flights.RunAsync("k", release, static (r, _) => new ValueTask(r.Task), Ct).AsTask()) + .ToArray(); + release.SetException(new InvalidOperationException("boom")); + + foreach (var caller in callers) + { + await caller.Invoking(c => c).Should().ThrowAsync(); + } + } + + [Fact] + public async Task A_caller_that_cancels_leaves_the_run_going_for_the_others() + { + var flights = new InFlight(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var workToken = CancellationToken.None; + using var leaving = new CancellationTokenSource(); + + var leaver = flights.RunAsync("k", release, (r, run) => { workToken = run.Token; return new ValueTask(r.Task); }, leaving.Token).AsTask(); + var stayer = flights.RunAsync("k", release, static (r, _) => new ValueTask(r.Task), Ct).AsTask(); + await leaving.CancelAsync(); + + await leaver.Invoking(t => t).Should().ThrowAsync(); + workToken.IsCancellationRequested.Should().BeFalse(); + release.SetResult(7); + (await stayer).Should().Be(7); + } + + [Fact] + public async Task The_run_is_cancelled_once_every_caller_has_cancelled() + { + var flights = new InFlight(); + var cancelled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var first = new CancellationTokenSource(); + using var second = new CancellationTokenSource(); + static async ValueTask UntilCancelled(TaskCompletionSource seen, IInFlightRun run) + { + await using (run.Token.Register(() => seen.TrySetResult())) + { + await Task.Delay(Timeout.Infinite, run.Token); + return 0; + } + } + + var a = flights.RunAsync("k", cancelled, UntilCancelled, first.Token).AsTask(); + var b = flights.RunAsync("k", cancelled, UntilCancelled, second.Token).AsTask(); + await first.CancelAsync(); + cancelled.Task.IsCompleted.Should().BeFalse("one caller is still waiting"); + await second.CancelAsync(); + + await cancelled.Task.WaitAsync(TimeSpan.FromSeconds(5), Ct); + await a.Invoking(t => t).Should().ThrowAsync(); + await b.Invoking(t => t).Should().ThrowAsync(); + } + + [Fact] + public async Task A_run_every_caller_left_leaves_the_table_even_if_its_work_ignores_cancellation() + { + var flights = new InFlight(); + var never = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var leaving = new CancellationTokenSource(); + + var abandoned = flights.RunAsync("k", never, static (n, _) => new ValueTask(n.Task), leaving.Token).AsTask(); + await leaving.CancelAsync(); + await abandoned.Invoking(t => t).Should().ThrowAsync(); + + flights.Count.Should().Be(0); + } + + [Fact] + public async Task A_caller_after_everyone_left_starts_a_fresh_run() + { + var flights = new InFlight(); + var runs = 0; + using var leaving = new CancellationTokenSource(); + async ValueTask Hang(int _, IInFlightRun run) + { + Interlocked.Increment(ref runs); + await Task.Delay(Timeout.Infinite, run.Token); + return 0; + } + + var abandoned = flights.RunAsync("k", 0, Hang, leaving.Token).AsTask(); + await leaving.CancelAsync(); + await abandoned.Invoking(t => t).Should().ThrowAsync(); + + var fresh = await flights.RunAsync("k", 0, (_, _) => new ValueTask(Interlocked.Increment(ref runs)), Ct); + + fresh.Should().Be(2); + } + + // Not inlined, so nothing in the calling frame keeps the flight reachable when the test collects. + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static async Task AbandonAFailingRunAsync() + { + var flights = new InFlight(); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var failing = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var leaving = new CancellationTokenSource(); + async ValueTask FailLate(TaskCompletionSource gate, IInFlightRun run) + { + await gate.Task; + failing.SetResult(); + throw new InvalidOperationException(AbandonedFailure); + } + + var abandoned = flights.RunAsync("k", release, FailLate, leaving.Token).AsTask(); + await leaving.CancelAsync(); + await abandoned.Invoking(t => t).Should().ThrowAsync(); + release.SetResult(); + await failing.Task; + await Task.Delay(100, Ct); + } +} diff --git a/tests/UiPath.Caching.Tests/InnerTierOutageTests.cs b/tests/UiPath.Caching.Tests/InnerTierOutageTests.cs new file mode 100644 index 0000000..0030fc7 --- /dev/null +++ b/tests/UiPath.Caching.Tests/InnerTierOutageTests.cs @@ -0,0 +1,542 @@ +#pragma warning disable CA2012 // NSubstitute setups call the ValueTask-returning members without consuming the result. +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging.Abstractions; +using UiPath.Caching.Policies; +using UiPath.Caching.Telemetry; +using UiPath.Caching.Tests.Fakes; + +namespace UiPath.Caching.Tests; + +/// Reads that share one inner read, values kept when the inner tier refuses them, and the local tier across an outage. +public class InnerTierOutageTests +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + // A waiter past the lock timeout runs the generator itself, which a loaded machine would otherwise reach. + private static InMemoryCacheOptions WaitingOutTheGenerator => new() { LocalLockTimeout = TimeSpan.FromSeconds(30) }; + + [Fact] + public async Task Concurrent_cold_reads_of_one_key_share_one_inner_read() + { + var inner = Substitute.For(); + var release = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var reads = 0; + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => { Interlocked.Increment(ref reads); return new ValueTask>(release.Task); }); + using var cache = InMemoryMultilayer.Cache(inner: inner); + + var callers = Enumerable.Range(0, 10).Select(_ => cache.GetAsync("user:42", policy: null, Ct).AsTask()).ToArray(); + release.SetResult(new TestCacheEntry { Value = "v", Expiration = DateTimeOffset.UtcNow.AddMinutes(5) }); + + (await Task.WhenAll(callers)).Should().AllBe("v"); + reads.Should().Be(1); + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v", "the shared read keeps the hit locally"); + reads.Should().Be(1); + } + + [Fact] + public async Task Concurrent_cold_reads_with_different_policies_read_separately() + { + var inner = Substitute.For(); + var release = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var reads = 0; + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => { Interlocked.Increment(ref reads); return new ValueTask>(release.Task); }); + using var cache = InMemoryMultilayer.Cache(inner: inner); + + var shortLived = cache.GetAsync("user:42", new CachePolicy { LocalExpiration = TimeSpan.FromSeconds(1) }, Ct).AsTask(); + var longLived = cache.GetAsync("user:42", new CachePolicy { LocalExpiration = TimeSpan.FromMinutes(1) }, Ct).AsTask(); + release.SetResult(new TestCacheEntry { Value = "v", Expiration = DateTimeOffset.UtcNow.AddMinutes(5) }); + + (await shortLived, await longLived).Should().Be(("v", "v")); + reads.Should().Be(2, "each keeps the hit with its own policy's local lifetime"); + } + + [Fact] + public async Task Concurrent_cold_reads_with_equivalent_policies_share_one_inner_read() + { + var inner = Substitute.For(); + var release = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var reads = 0; + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => { Interlocked.Increment(ref reads); return new ValueTask>(release.Task); }); + using var cache = InMemoryMultilayer.Cache(inner: inner); + + var a = cache.GetAsync("user:42", new CachePolicy { LocalExpiration = TimeSpan.FromMinutes(1) }, Ct).AsTask(); + var b = cache.GetAsync("user:42", new CachePolicy { LocalExpiration = TimeSpan.FromMinutes(1) }, Ct).AsTask(); + release.SetResult(new TestCacheEntry { Value = "v", Expiration = DateTimeOffset.UtcNow.AddMinutes(5) }); + + (await a, await b).Should().Be(("v", "v")); + reads.Should().Be(1, "two policies built separately with the same local lifetime read alike"); + } + + [Fact] + public async Task Concurrent_cold_hash_reads_of_different_fields_share_one_inner_read() + { + var inner = Substitute.For(); + var release = new TaskCompletionSource>>(TaskCreationOptions.RunContinuationsAsynchronously); + var reads = 0; + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => { Interlocked.Increment(ref reads); return new ValueTask>>(release.Task); }); + using var cache = InMemoryMultilayer.HashCache(inner: inner); + + var a = cache.GetItemAsync("user:42", "a", policy: null, Ct).AsTask(); + var b = cache.GetItemAsync("user:42", "b", policy: null, Ct).AsTask(); + release.SetResult(new TestCacheEntry> + { + Value = new Dictionary { ["a"] = "1", ["b"] = "2" }, + Expiration = DateTimeOffset.UtcNow.AddMinutes(5), + }); + + (await a, await b).Should().Be(("1", "2")); + reads.Should().Be(1); + } + + [Fact] + public async Task Concurrent_misses_run_the_generator_once_when_the_inner_tier_refuses_the_write() + { + var inner = Substitute.For(); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ValueTask>(new TestCacheEntry { Expiration = DateTimeOffset.MinValue })); + inner.SetAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(false); + using var cache = InMemoryMultilayer.Cache(WaitingOutTheGenerator, inner: inner); + var generated = 0; + async Task Generate(CancellationToken token) + { + Interlocked.Increment(ref generated); + await Task.Delay(50, token); + return "fresh"; + } + + var callers = Enumerable.Range(0, 8).Select(_ => cache.GetOrAddAsync("user:42", Generate, policy: null, Ct).AsTask()).ToArray(); + + (await Task.WhenAll(callers)).Should().AllBe("fresh"); + generated.Should().Be(1, "the callers waiting on the local lock reuse the value the inner tier refused"); + } + + [Fact] + public async Task Concurrent_hash_misses_run_the_generator_once_when_the_inner_tier_refuses_the_write() + { + var inner = Substitute.For(); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ValueTask>>(new TestCacheEntry> { Expiration = DateTimeOffset.MinValue })); + inner.SetAsync(Arg.Any(), Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(false); + using var cache = InMemoryMultilayer.HashCache(WaitingOutTheGenerator, inner: inner); + var generated = 0; + async Task> Generate(CancellationToken token) + { + Interlocked.Increment(ref generated); + await Task.Delay(50, token); + return new Dictionary { ["f"] = "fresh" }; + } + + var callers = Enumerable.Range(0, 8).Select(_ => cache.GetOrAddAsync("user:42", Generate, policy: null, Ct).AsTask()).ToArray(); + + (await Task.WhenAll(callers)).Should().AllSatisfy(values => values.Should().ContainKey("f")); + generated.Should().Be(1, "the callers waiting on the local lock reuse the value the inner tier refused"); + } + + [Fact] + public async Task Concurrent_misses_run_the_generator_once_when_the_inner_write_throws() + { + var inner = Substitute.For(); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ValueTask>(new TestCacheEntry { Expiration = DateTimeOffset.MinValue })); + inner.SetAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns>(_ => throw new InvalidOperationException("inner write failed")); + using var cache = InMemoryMultilayer.Cache(WaitingOutTheGenerator, inner: inner); + var generated = 0; + async Task Generate(CancellationToken token) + { + Interlocked.Increment(ref generated); + await Task.Delay(50, token); + return "fresh"; + } + + var callers = Enumerable.Range(0, 8).Select(_ => cache.GetOrAddAsync("user:42", Generate, policy: null, Ct).AsTask()).ToArray(); + + (await Task.WhenAll(callers)).Should().AllBe("fresh"); + generated.Should().Be(1, "a write that throws is refused like one that returns false"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Concurrent_batch_misses_run_the_generator_once_when_the_inner_tier_refuses_the_write(bool throws) + { + var inner = BatchMissInner(); + inner.SetAsync(Arg.Any[]>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => throws ? throw new InvalidOperationException("inner write failed") : new ValueTask(false)); + using var cache = InMemoryMultilayer.Cache(WaitingOutTheGenerator, inner: inner); + var generated = 0; + async Task[]> Generate(int[] states, CancellationToken token) + { + Interlocked.Increment(ref generated); + await Task.Delay(50, token); + return Array.ConvertAll(states, s => new KeyValuePair(s, $"fresh{s}")); + } + + var callers = Enumerable.Range(0, 8) + .Select(_ => cache.GetOrAddAsync([new("user:1", 1), new("user:2", 2)], Generate, policy: null, Ct).AsTask()) + .ToArray(); + + (await Task.WhenAll(callers)).Should().AllSatisfy(values => values.Select(v => v.Value).Should().Equal("fresh1", "fresh2")); + generated.Should().Be(1, "the callers waiting on the local lock reuse the values the inner tier refused"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_batch_miss_canceled_during_the_inner_write_keeps_nothing_local(bool answersFalse) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(Ct); + var inner = BatchMissInner(); + inner.SetAsync(Arg.Any[]>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => CancelWrite(cts, answersFalse), _ => new ValueTask(false)); + using var cache = InMemoryMultilayer.Cache(inner: inner); + var generated = 0; + Task[]> Generate(int[] states, CancellationToken _) + { + Interlocked.Increment(ref generated); + return Task.FromResult(Array.ConvertAll(states, s => new KeyValuePair(s, "v"))); + } + + await IgnoreCancellation(() => cache.GetOrAddAsync([new("user:1", 1)], Generate, policy: null, cts.Token).AsTask()); + await cache.GetOrAddAsync([new("user:1", 1)], Generate, policy: null, Ct); + + generated.Should().Be(2, "the canceled call is not a refusal, so it leaves no local copy behind"); + } + + // A Redis setter answers a write canceled under it with false rather than throwing. + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(false, true)] + [InlineData(true, true)] + public async Task A_miss_canceled_during_the_inner_write_keeps_nothing_local(bool hash, bool answersFalse) + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(Ct); + var generated = 0; + + if (hash) + { + var inner = Substitute.For(); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ValueTask>>(new TestCacheEntry> { Expiration = DateTimeOffset.MinValue })); + inner.SetAsync(Arg.Any(), Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => CancelWrite(cts, answersFalse), _ => new ValueTask(false)); + using var cache = InMemoryMultilayer.HashCache(inner: inner); + Task> Generate(CancellationToken _) + { + Interlocked.Increment(ref generated); + return Task.FromResult>(new Dictionary { ["f"] = "v" }); + } + + await IgnoreCancellation(() => cache.GetOrAddAsync("user:42", Generate, policy: null, cts.Token).AsTask()); + await cache.GetOrAddAsync("user:42", Generate, policy: null, Ct); + } + else + { + var inner = Substitute.For(); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ValueTask>(new TestCacheEntry { Expiration = DateTimeOffset.MinValue })); + inner.SetAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => CancelWrite(cts, answersFalse), _ => new ValueTask(false)); + using var cache = InMemoryMultilayer.Cache(inner: inner); + Task Generate(CancellationToken _) + { + Interlocked.Increment(ref generated); + return Task.FromResult("v"); + } + + await IgnoreCancellation(() => cache.GetOrAddAsync("user:42", Generate, policy: null, cts.Token).AsTask()); + await cache.GetOrAddAsync("user:42", Generate, policy: null, Ct); + } + + generated.Should().Be(2, "the canceled call is not a refusal, so it leaves no local copy behind"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_local_write_failure_after_an_accepted_inner_write_is_not_retried(bool hash) + { + var sizes = 0; + var sizeProvider = Substitute.For(); + sizeProvider.GetSize(Arg.Any()).Returns(_ => + { + Interlocked.Increment(ref sizes); + throw new InvalidOperationException("local write failed"); + }); + var options = new InMemoryCacheOptions { SizeLimit = 1_000, SizeProvider = sizeProvider }; + if (hash) + { + var inner = Substitute.For(); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ValueTask>>(new TestCacheEntry> { Expiration = DateTimeOffset.MinValue })); + inner.SetAsync(Arg.Any(), Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(true); + using var cache = InMemoryMultilayer.HashCache(options, inner); + await cache.GetOrAddAsync("user:42", _ => Task.FromResult>(new Dictionary { ["f"] = "v" }), policy: null, Ct); + } + else + { + var inner = Substitute.For(); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ValueTask>(new TestCacheEntry { Expiration = DateTimeOffset.MinValue })); + inner.SetAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(true); + using var cache = InMemoryMultilayer.Cache(options, inner); + await cache.GetOrAddAsync("user:42", _ => Task.FromResult("v"), policy: null, Ct); + } + + sizes.Should().Be(1, "the inner tier took the write, so the disconnected-cap fallback does not apply"); + } + + [Fact] + public async Task A_cold_read_during_an_outage_is_kept_for_the_disconnected_cap() + { + var inner = Substitute.For(); + var state = (IConnectionState)inner; + state.IsConnected.Returns(false); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => new ValueTask>(new TestCacheEntry { Value = "v", Expiration = DateTimeOffset.UtcNow.AddHours(1) })); + var now = DateTimeOffset.UtcNow; + var systemClock = Substitute.For(); + systemClock.UtcNow.Returns(_ => now); + var clock = new SystemClockTimeProvider(systemClock); + using var cache = InMemoryMultilayer.Cache(new InMemoryCacheOptions { LocalMaxExpirationDisconnected = TimeSpan.FromSeconds(30), LocalMaxExpiration = TimeSpan.FromHours(1) }, inner, connectionMonitor: true, clock: clock); + state.OnConnectionFailed += Raise.Event(state, EventArgs.Empty); + + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v"); + now = now.AddMinutes(1); + inner.ClearReceivedCalls(); + await cache.GetAsync("user:42", policy: null, Ct); + + await inner.Received(1).GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task Concurrent_cold_reads_with_different_disconnected_caps_read_separately() + { + var inner = Substitute.For(); + var release = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); + var reads = 0; + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => { Interlocked.Increment(ref reads); return new ValueTask>(release.Task); }); + using var cache = InMemoryMultilayer.Cache(inner: inner); + + var shortCap = cache.GetAsync("user:42", new CachePolicy { LocalExpirationDisconnected = TimeSpan.FromSeconds(1) }, Ct).AsTask(); + var longCap = cache.GetAsync("user:42", new CachePolicy { LocalExpirationDisconnected = TimeSpan.FromMinutes(1) }, Ct).AsTask(); + release.SetResult(new TestCacheEntry { Value = "v", Expiration = DateTimeOffset.UtcNow.AddMinutes(5) }); + + (await shortCap, await longCap).Should().Be(("v", "v")); + reads.Should().Be(2, "a read that finds the tier down keeps the hit for its own policy's disconnected cap"); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task A_cold_batch_read_during_an_outage_is_kept_for_the_disconnected_cap(bool entries) + { + var inner = Substitute.For(); + var state = (IConnectionState)inner; + state.IsConnected.Returns(false); + inner.GetCacheEntriesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => new ValueTask>[]>(call.Arg() + .Select(k => new KeyValuePair>(k, new TestCacheEntry { Value = "v", Expiration = DateTimeOffset.UtcNow.AddHours(1) })) + .ToArray())); + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => new ValueTask>(new TestCacheEntry { Value = "v", Expiration = DateTimeOffset.UtcNow.AddHours(1) })); + var now = DateTimeOffset.UtcNow; + var systemClock = Substitute.For(); + systemClock.UtcNow.Returns(_ => now); + var clock = new SystemClockTimeProvider(systemClock); + using var cache = InMemoryMultilayer.Cache(new InMemoryCacheOptions { LocalMaxExpirationDisconnected = TimeSpan.FromSeconds(30), LocalMaxExpiration = TimeSpan.FromHours(1) }, inner, connectionMonitor: true, clock: clock); + state.OnConnectionFailed += Raise.Event(state, EventArgs.Empty); + CacheKey[] keys = ["user:1", "user:2"]; + + if (entries) + { + await cache.GetCacheEntriesAsync(keys, policy: null, Ct); + } + else + { + await cache.GetAsync(keys, policy: null, Ct); + } + + (await cache.GetAsync("user:1", policy: null, Ct)).Should().Be("v"); + await inner.DidNotReceive().GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()); + now = now.AddMinutes(1); + await cache.GetAsync("user:1", policy: null, Ct); + + await inner.Received(1).GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task A_disconnected_tier_serves_the_local_copy_by_default() + { + var inner = Substitute.For(); + var state = (IConnectionState)inner; + state.IsConnected.Returns(true); + inner.SetAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + using var cache = InMemoryMultilayer.Cache(inner: inner, connectionMonitor: true); + (await cache.SetAsync("user:42", "v", policy: null, Ct)).Should().BeTrue(); + + state.IsConnected.Returns(false); + state.OnConnectionFailed += Raise.Event(state, EventArgs.Empty); + + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v"); + } + + [Theory] + [InlineData(null, MissedEventsReason.SubscriptionGap, true)] + [InlineData(false, MissedEventsReason.SubscriptionGap, false)] + [InlineData(false, MissedEventsReason.Lost, true)] + public async Task A_cache_that_keeps_its_local_tier_on_reconnect_keeps_it_over_a_subscription_gap_too(bool? clearLocalOnReconnect, MissedEventsReason reason, bool expired) + { + IObserver? observer = null; + var topic = Substitute.For>(); + topic.Subscribe(Arg.Do>(o => observer = o)).Returns(Substitute.For()); + var broadcast = Substitute.For(); + broadcast.Create(Arg.Any()).Returns(topic); + ((IConnectionState)broadcast).IsConnected.Returns(true); + var tokens = new ChangeTokenFactory( + Options.Create(new CacheOptions { AppShortName = "test", SourceUri = new Uri("urn:node") }), + Substitute.For>(), + NullLoggerFactory.Instance, + NullTelemetryProvider.Instance); + using var cache = InMemoryMultilayer.Cache(new InMemoryCacheOptions { ClearLocalOnReconnect = clearLocalOnReconnect }, ReadOnce(), connectionMonitor: true, topics: Topics(broadcast), tokens: tokens); + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v"); + + ((IMissedEventsObserver)observer!).OnEventsMissed(reason); + + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be(expired ? null : "v"); + } + + [Theory] + [InlineData(null, MissedEventsReason.SubscriptionGap, true)] + [InlineData(false, MissedEventsReason.SubscriptionGap, false)] + [InlineData(false, MissedEventsReason.Lost, true)] + public async Task A_custom_token_factory_keeps_entries_over_a_subscription_gap_when_the_cache_does(bool? clearLocalOnReconnect, MissedEventsReason reason, bool expired) + { + IObserver? observer = null; + var topic = Substitute.For>(); + topic.Subscribe(Arg.Do>(o => observer = o)).Returns(Substitute.For()); + var broadcast = Substitute.For(); + broadcast.Create(Arg.Any()).Returns(topic); + ((IConnectionState)broadcast).IsConnected.Returns(true); + var tokens = new TestChangeTokenFactory((key, t) => new ChangeToken( + key, + t, + new Uri("urn:node"), + Substitute.For>(), + NullLogger>.Instance, + NullTelemetryProvider.Instance)); + using var cache = InMemoryMultilayer.Cache(new InMemoryCacheOptions { ClearLocalOnReconnect = clearLocalOnReconnect }, ReadOnce(), connectionMonitor: true, topics: Topics(broadcast), tokens: tokens); + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v"); + + ((IMissedEventsObserver)observer!).OnEventsMissed(reason); + + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be(expired ? null : "v"); + } + + [Fact] + public async Task A_restored_connection_keeps_the_local_tier_when_no_broadcast_could_be_missed() + { + var inner = Substitute.For(); + var state = (IConnectionState)inner; + state.IsConnected.Returns(true); + StubReadOnce(inner); + using var cache = InMemoryMultilayer.Cache(inner: inner, connectionMonitor: true); + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v"); + + state.IsConnected.Returns(false); + state.OnConnectionFailed += Raise.Event(state, EventArgs.Empty); + state.IsConnected.Returns(true); + state.OnConnectionRestored += Raise.Event(state, EventArgs.Empty); + + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v"); + } + + [Fact] + public async Task A_restored_connection_keeps_the_local_tier_when_broadcast_runs_over_streams() + { + var connector = Substitute.For(); + connector.IsConnected.Returns(true); + using var streams = StreamsProvider(connector); + using var cache = InMemoryMultilayer.Cache(inner: ReadOnce(), connectionMonitor: true, topics: Topics(streams)); + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v"); + + connector.IsConnected.Returns(false); + connector.OnConnectionFailed += Raise.Event(connector, EventArgs.Empty); + connector.IsConnected.Returns(true); + connector.OnConnectionRestored += Raise.Event(connector, EventArgs.Empty); + + (await cache.GetAsync("user:42", policy: null, Ct)).Should().Be("v", "the stream replays what the outage held back"); + } + + [Fact] + public void The_connection_monitor_is_on_by_default() => + new CacheOptions().ConnectionMonitorEnabled.Should().BeTrue(); + + private static async Task IgnoreCancellation(Func call) + { + try + { + await call(); + } + catch (OperationCanceledException) + { + } + } + + /// An inner tier that holds the value for the first read only, so a later hit can only come from the local tier. + private static ICache ReadOnce() + { + var inner = Substitute.For(); + StubReadOnce(inner); + return inner; + } + + private static void StubReadOnce(ICache inner) => + inner.GetCacheEntryAsync(Arg.Any(), Arg.Any(), Arg.Any()).Returns( + new ValueTask>(new TestCacheEntry { Value = "v", Expiration = DateTimeOffset.UtcNow.AddMinutes(5) }), + new ValueTask>(new TestCacheEntry { Expiration = DateTimeOffset.MinValue })); + + private static ITopicFactory Topics(ITopicProvider provider) + { + var topics = Substitute.For(); + topics.Get(Arg.Any()).Returns(provider); + return topics; + } + + private static RedisStreamsTopicProvider StreamsProvider(IRedisConnector connector) => new( + Options.Create(new RedisStreamsTopicOptions { Enabled = true, ConnectionMonitorEnabled = true }), + Options.Create(new CacheOptions { AppShortName = "test" }), + new PerTopicOptionsRegistry(new ConfigurationBuilder().Build().GetSection("Topics")), + connector, + Substitute.For>(), + Substitute.For(), + NullLoggerFactory.Instance, + NullTelemetryProvider.Instance, + NullRedisProfiler.Instance); + + private static ValueTask CancelWrite(CancellationTokenSource cts, bool answersFalse) + { + cts.Cancel(); + return answersFalse ? new ValueTask(false) : throw new OperationCanceledException(cts.Token); + } + + private static ICache BatchMissInner() + { + var inner = Substitute.For(); + inner.GetCacheEntriesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ci => new ValueTask>[]>(Array.ConvertAll( + ci.Arg(), + k => new KeyValuePair>(k, new TestCacheEntry { Expiration = DateTimeOffset.MinValue })))); + return inner; + } +} diff --git a/tests/UiPath.Caching.Tests/MultilayerCachePerNamePolicyWiringTests.cs b/tests/UiPath.Caching.Tests/MultilayerCachePerNamePolicyWiringTests.cs index 3351fe7..c667e4a 100644 --- a/tests/UiPath.Caching.Tests/MultilayerCachePerNamePolicyWiringTests.cs +++ b/tests/UiPath.Caching.Tests/MultilayerCachePerNamePolicyWiringTests.cs @@ -564,6 +564,8 @@ public ValueTask InitializeAsync() _distributedLock = _fixture.Freeze(); _options = new InMemoryRedisCacheOptions { + // These tests assume a tier that is never seen as disconnected; the ones about disconnection turn the monitor on. + ConnectionMonitorEnabled = false, DefaultExpiration = TimeSpan.FromMinutes(10), EntryFactory = new TestCacheEntryFactory(), LocalLockEnabled = false, diff --git a/tests/UiPath.Caching.Tests/MultilayerCacheTests.cs b/tests/UiPath.Caching.Tests/MultilayerCacheTests.cs index fc21f3e..eb330da 100644 --- a/tests/UiPath.Caching.Tests/MultilayerCacheTests.cs +++ b/tests/UiPath.Caching.Tests/MultilayerCacheTests.cs @@ -415,7 +415,8 @@ public async Task GetOrAdd_data_from_generator(bool innerCacheSet) var actual = await Sut.GetOrAddAsync(_cacheKey, generator, _fixture.Create(), token: testContextAccessor.Current.CancellationToken); generatorWasCalled.Should().BeTrue(); - _memoryCache.Received(innerCacheSet ? 1 : 0).CreateEntry(_innerCacheKey.Name); + // A value the inner tier refused is still kept locally, for the disconnected lifetime, so waiters on the lock reuse it. + _memoryCache.Received(1).CreateEntry(_innerCacheKey.Name); await _innerCache.Received(1).SetAsync(_innerCacheKey, Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); actual.Should().Be(generatorExpected); _logger.ReceivedCalls().Should().Contain(c => c.GetMethodInfo().Name == "Log" && (LogLevel)c.GetArguments()[0]! == LogLevel.Debug); @@ -1460,6 +1461,8 @@ public ValueTask InitializeAsync() _clock = new SystemClock(); _options = new() { + // These tests assume a tier that is never seen as disconnected; the ones about disconnection turn the monitor on. + ConnectionMonitorEnabled = false, DefaultExpiration = TimeSpan.FromMinutes(10), EntryFactory = new TestCacheEntryFactory(), }; diff --git a/tests/UiPath.Caching.Tests/MultilayerHashCachePerNameJitterTests.cs b/tests/UiPath.Caching.Tests/MultilayerHashCachePerNameJitterTests.cs index 8172a55..ff1fb61 100644 --- a/tests/UiPath.Caching.Tests/MultilayerHashCachePerNameJitterTests.cs +++ b/tests/UiPath.Caching.Tests/MultilayerHashCachePerNameJitterTests.cs @@ -248,6 +248,8 @@ public ValueTask InitializeAsync() _distributedLock = _fixture.Freeze(); _options = new InMemoryRedisCacheOptions { + // These tests assume a tier that is never seen as disconnected; the ones about disconnection turn the monitor on. + ConnectionMonitorEnabled = false, DefaultExpiration = TimeSpan.FromMinutes(10), EntryFactory = new TestCacheEntryFactory(), LocalLockEnabled = false, diff --git a/tests/UiPath.Caching.Tests/MultilayerHashCacheRehydrateTests.cs b/tests/UiPath.Caching.Tests/MultilayerHashCacheRehydrateTests.cs index 0cf26c0..896ab8c 100644 --- a/tests/UiPath.Caching.Tests/MultilayerHashCacheRehydrateTests.cs +++ b/tests/UiPath.Caching.Tests/MultilayerHashCacheRehydrateTests.cs @@ -306,6 +306,8 @@ public ValueTask InitializeAsync() _distributedLock = _fixture.Freeze(); _options = new InMemoryRedisCacheOptions { + // These tests assume a tier that is never seen as disconnected; the ones about disconnection turn the monitor on. + ConnectionMonitorEnabled = false, DefaultExpiration = Duration, EntryFactory = new TestCacheEntryFactory(), LocalLockEnabled = false, diff --git a/tests/UiPath.Caching.Tests/MultilayerHashCacheTests.cs b/tests/UiPath.Caching.Tests/MultilayerHashCacheTests.cs index 9c15b2c..3e60f00 100644 --- a/tests/UiPath.Caching.Tests/MultilayerHashCacheTests.cs +++ b/tests/UiPath.Caching.Tests/MultilayerHashCacheTests.cs @@ -307,7 +307,8 @@ public async Task GetOrAdd_data_from_generator(bool innerCacheSet) .Returns(expectedCacheEntry); var actual = await Sut.GetOrAddAsync(_cacheKey, generator, _fixture.Create(), token: testContextAccessor.Current.CancellationToken); generatorWasCalled.Should().BeTrue(); - _memoryCache.Received(innerCacheSet ? 1 : 0).CreateEntry(_innerCacheKey.Name); + // A value the inner tier refused is still kept locally, for the disconnected lifetime, so waiters on the lock reuse it. + _memoryCache.Received(1).CreateEntry(_innerCacheKey.Name); await _innerCache.Received(1).SetAsync(_innerCacheKey, Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()); actual.Should().BeEquivalentTo(generatorExpected); } @@ -1472,6 +1473,10 @@ public ValueTask InitializeAsync() _options = new() { + + // These tests assume a tier that is never seen as disconnected; the ones about disconnection turn the monitor on. + + ConnectionMonitorEnabled = false, DefaultExpiration = TimeSpan.FromMinutes(10), EntryFactory = new TestCacheEntryFactory(), CacheKeyStrategy = _cacheKeyStrategy, diff --git a/tests/UiPath.Caching.Tests/Redis/ConnectionStateMonitorRecoveredTests.cs b/tests/UiPath.Caching.Tests/Redis/ConnectionStateMonitorRecoveredTests.cs new file mode 100644 index 0000000..cd70b92 --- /dev/null +++ b/tests/UiPath.Caching.Tests/Redis/ConnectionStateMonitorRecoveredTests.cs @@ -0,0 +1,437 @@ +using UiPath.Caching.Redis; +using UiPath.Caching.Tests.Telemetry; + +namespace UiPath.Caching.Tests.Redis; + +public class ConnectionStateMonitorRecoveredTests +{ + private readonly RecordingTelemetryProvider _telemetry = new(); + + [Fact] + public void A_restore_while_another_state_is_still_down_is_not_a_recovery() + { + var inner = Connected(); + var broadcast = Connected(); + using var sut = new ConnectionStateMonitor(_telemetry, Timeout.InfiniteTimeSpan, inner, broadcast); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + + Fail(inner); + Fail(broadcast); + Restore(inner); + recovered.Should().Be(0, "the broadcast connection is still down"); + + Restore(broadcast); + recovered.Should().Be(1); + } + + [Fact] + public void A_state_down_at_creation_recovers_on_its_first_restore() + { + var inner = Connected(); + inner.IsConnected.Returns(false); + using var sut = new ConnectionStateMonitor(_telemetry, Timeout.InfiniteTimeSpan, inner); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + + Restore(inner); + + recovered.Should().Be(1); + } + + [Fact] + public void One_restore_forwarded_by_two_facades_is_one_recovery() + { + var connector = Connected(); + using var sut = new ConnectionStateMonitor(_telemetry, Timeout.InfiniteTimeSpan, connector, connector); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + + Fail(connector); + Restore(connector); + + recovered.Should().Be(1, "both subscriptions hear the same restore"); + } + + [Fact] + public void A_forced_reconnect_without_an_outage_is_a_recovery() + { + var inner = Connected(); + using var sut = new ConnectionStateMonitor(_telemetry, Timeout.InfiniteTimeSpan, inner); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + + inner.OnReconnected += Raise.Event(inner, new EventArgs()); + + recovered.Should().Be(1, "the swap can drop publications although the connection never read as down"); + } + + [Fact] + public void Every_forced_reconnect_is_a_recovery_whatever_args_it_carries() + { + var inner = Connected(); + using var sut = new ConnectionStateMonitor(_telemetry, Timeout.InfiniteTimeSpan, inner); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + var reused = new EventArgs(); + + inner.OnReconnected += Raise.Event(inner, reused); + inner.OnReconnected += Raise.Event(inner, reused); + inner.OnReconnected += Raise.Event(inner, EventArgs.Empty); + + recovered.Should().Be(3, "a source may reuse one args instance, so it says nothing about the swap"); + } + + [Fact] + public void A_recovery_found_by_polling_is_raised_once() + { + var inner = Connected(); + var timers = new ManualTimers(); + using var sut = new ConnectionStateMonitor(_telemetry, TimeSpan.FromMilliseconds(20), timers, inner); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + + Fail(inner); + sut.IsConnected.Should().BeFalse(); + inner.IsConnected.Returns(true); + timers.Tick(); + timers.Tick(); + + recovered.Should().Be(1); + timers.Armed.Should().Be(0, "the poll stops once recovery holds"); + } + + [Fact] + public void An_outage_found_only_by_polling_is_seen_to_end() + { + var inner = Connected(); + inner.IsConnected.Returns(false); + var timers = new ManualTimers(); + using var sut = new ConnectionStateMonitor(_telemetry, TimeSpan.FromMilliseconds(20), timers, inner); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + timers.Tick(); + + inner.IsConnected.Returns(true); + timers.Tick(); + + recovered.Should().Be(1); + } + + [Fact] + public void A_zero_monitor_interval_still_polls_until_recovery() + { + var inner = Connected(); + inner.IsConnected.Returns(false); + var timers = new ManualTimers(); + using var sut = new ConnectionStateMonitor(_telemetry, TimeSpan.Zero, timers, inner); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + sut.IsConnected.Should().BeFalse(); + + timers.Period.Should().Be(TimeSpan.FromSeconds(5), "a zero period would fire once and could miss the recovery for good"); + inner.IsConnected.Returns(true); + timers.Tick(); + + recovered.Should().Be(1); + } + + [Fact] + public void A_disposed_monitor_stops_polling_however_many_failures_replaced_its_timer() + { + var inner = Connected(); + var timers = new ManualTimers(); + var sut = new ConnectionStateMonitor(_telemetry, TimeSpan.FromMilliseconds(10), timers, inner); + Fail(inner); + Fail(inner); + Fail(inner); + timers.Armed.Should().Be(1, "each failure stops the timer it replaces"); + + sut.Dispose(); + + timers.Armed.Should().Be(0, "every timer the failures created was stopped"); + } + + [Fact] + public async Task A_poll_that_sampled_an_outage_before_a_restore_does_not_recover_twice() + { + var state = new HeldState(); + var timers = new ManualTimers(); + using var sut = new ConnectionStateMonitor(_telemetry, TimeSpan.FromMilliseconds(20), timers, state); + var recovered = 0; + sut.Recovered += (_, _) => Interlocked.Increment(ref recovered); + state.Fail(); + var sampling = state.HoldNextRead(); + var poll = Task.Run(timers.Tick, TestContext.Current.CancellationToken); + sampling.Wait(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).Should().BeTrue("the poll the failure armed reads the state"); + + state.Restore(); + Volatile.Read(ref recovered).Should().Be(1); + state.Release(); + await poll; + timers.Tick(); + + Volatile.Read(ref recovered).Should().Be(1, "the held poll sampled the outage the restore already ended"); + } + + [Fact] + public async Task A_poll_whose_healthy_sample_went_stale_keeps_polling() + { + var state = new HeldState(); + var timers = new ManualTimers(); + using var sut = new ConnectionStateMonitor(_telemetry, TimeSpan.FromMilliseconds(20), timers, state); + var recovered = 0; + sut.Recovered += (_, _) => Interlocked.Increment(ref recovered); + state.Fail(); + var sampling = state.HoldNextRead(reading: true); + var poll = Task.Run(timers.Tick, TestContext.Current.CancellationToken); + sampling.Wait(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).Should().BeTrue(); + + state.Release(); + await poll; + Volatile.Read(ref recovered).Should().Be(0, "the final check found the state still down"); + state.SetSilently(up: true); + timers.Tick(); + + Volatile.Read(ref recovered).Should().Be(1); + } + + [Fact] + public async Task A_state_down_at_creation_restored_while_its_first_poll_is_held_recovers() + { + var state = new HeldState(up: false); + var sampling = state.HoldNextRead(); + var timers = new ManualTimers(); + using var sut = new ConnectionStateMonitor(_telemetry, TimeSpan.FromMilliseconds(20), timers, state); + var recovered = 0; + sut.Recovered += (_, _) => Interlocked.Increment(ref recovered); + var poll = Task.Run(timers.Tick, TestContext.Current.CancellationToken); + sampling.Wait(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).Should().BeTrue(); + + state.Restore(); + state.Release(); + await poll; + timers.Tick(); + + Volatile.Read(ref recovered).Should().Be(1, "the held poll had not recorded its sample, so the restore is the only sign of the outage"); + } + + [Fact] + public async Task A_state_read_that_sampled_an_outage_before_a_restore_does_not_recover_twice() + { + var state = new HeldState(); + using var sut = new ConnectionStateMonitor(_telemetry, Timeout.InfiniteTimeSpan, state); + var recovered = 0; + sut.Recovered += (_, _) => Interlocked.Increment(ref recovered); + state.Fail(); + var sampling = state.HoldNextRead(); + var read = Task.Run(() => sut.IsConnected, TestContext.Current.CancellationToken); + sampling.Wait(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken).Should().BeTrue(); + + state.Restore(); + recovered.Should().Be(1); + state.Release(); + (await read).Should().BeFalse(); + state.Restore(); + + recovered.Should().Be(1, "the held read sampled the outage the restore already ended"); + } + + [Fact] + public void A_poll_whose_final_check_finds_the_state_down_again_keeps_polling() + { + var state = new ScriptedState(); + var timers = new ManualTimers(); + using var sut = new ConnectionStateMonitor(_telemetry, TimeSpan.FromMilliseconds(20), timers, state); + var recovered = 0; + sut.Recovered += (_, _) => recovered++; + + // The poll samples twice, then the recovery check samples a third time and finds the state down again. + state.Fail(readsUntilDown: 2); + timers.Tick(); + recovered.Should().Be(0); + timers.Armed.Should().Be(1, "the timer keeps polling until recovery holds"); + timers.Tick(); + + recovered.Should().Be(1); + } + + private static IConnectionState Connected() + { + var state = Substitute.For(); + state.IsConnected.Returns(true); + return state; + } + + private static void Fail(IConnectionState state) + { + state.IsConnected.Returns(false); + state.OnConnectionFailed += Raise.Event(state, EventArgs.Empty); + } + + private static void Restore(IConnectionState state) + { + state.IsConnected.Returns(true); + state.OnConnectionRestored += Raise.Event(state, EventArgs.Empty); + } + + private int Evaluations() => _telemetry.Events.Count(e => e.Name == "Redis.EvaluateConnected"); + + /// A failed state that reads up a set number of times, then down once, then up for good. + private sealed class ScriptedState : IConnectionState + { + private int _readsUntilDown = -1; + + public event EventHandler? OnConnectionFailed; + + public event EventHandler? OnConnectionRestored + { + add { } + remove { } + } + + public event EventHandler? OnReconnected + { + add { } + remove { } + } + + public bool IsConnected => Interlocked.Decrement(ref _readsUntilDown) != -1; + + public void Fail(int readsUntilDown) + { + Volatile.Write(ref _readsUntilDown, int.MinValue / 2); + OnConnectionFailed?.Invoke(this, EventArgs.Empty); + Volatile.Write(ref _readsUntilDown, readsUntilDown); + } + } + + /// A state whose next read can be held mid-sample, reporting a set value, until released. + private sealed class HeldState : IConnectionState + { + private readonly ManualResetEventSlim _release = new(); + private ManualResetEventSlim? _sampling; + private bool _reading; + private volatile bool _up; + + public HeldState(bool up = true) => _up = up; + + public event EventHandler? OnConnectionFailed; + + public event EventHandler? OnConnectionRestored; + + public event EventHandler? OnReconnected; + + public bool IsConnected + { + get + { + var sampling = Interlocked.Exchange(ref _sampling, null); + if (sampling is null) + { + return _up; + } + + sampling.Set(); + _release.Wait(TimeSpan.FromSeconds(10)); + return _reading; + } + } + + public ManualResetEventSlim HoldNextRead(bool reading = false) + { + _reading = reading; + var sampling = new ManualResetEventSlim(); + _sampling = sampling; + return sampling; + } + + public void Release() => _release.Set(); + + public void SetSilently(bool up) => _up = up; + + public void Fail() + { + _up = false; + OnConnectionFailed?.Invoke(this, EventArgs.Empty); + } + + public void Restore() + { + _up = true; + OnConnectionRestored?.Invoke(this, EventArgs.Empty); + } + + public void Reconnect() => OnReconnected?.Invoke(this, EventArgs.Empty); + } + + /// Timers that fire only when a test ticks them, so no poll interleaves with the sequence it drives. + private sealed class ManualTimers : TimeProvider + { + private readonly List _timers = []; + + public int Armed + { + get + { + lock (_timers) + { + return _timers.Count(t => t.IsArmed); + } + } + } + + public TimeSpan Period { get; private set; } + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = new ManualTimer(this, callback, state); + lock (_timers) + { + _timers.Add(timer); + } + + return timer; + } + + /// Fires every armed timer once, as one period elapsing would. + public void Tick() + { + ManualTimer[] armed; + lock (_timers) + { + armed = [.. _timers.Where(t => t.IsArmed)]; + } + + foreach (var timer in armed) + { + timer.Fire(); + } + } + + private sealed class ManualTimer(ManualTimers owner, TimerCallback callback, object? state) : ITimer + { + private volatile bool _armed; + private volatile bool _disposed; + + public bool IsArmed => _armed && !_disposed; + + public bool Change(TimeSpan dueTime, TimeSpan period) + { + _armed = dueTime != Timeout.InfiniteTimeSpan; + owner.Period = period; + return !_disposed; + } + + public void Fire() => callback(state); + + public void Dispose() => _disposed = true; + + public ValueTask DisposeAsync() + { + Dispose(); + return default; + } + } + } +} diff --git a/tests/UiPath.Caching.Tests/Redis/RedisConnectorLifecycleTests.cs b/tests/UiPath.Caching.Tests/Redis/RedisConnectorLifecycleTests.cs index b39ecaa..8ca6cfb 100644 --- a/tests/UiPath.Caching.Tests/Redis/RedisConnectorLifecycleTests.cs +++ b/tests/UiPath.Caching.Tests/Redis/RedisConnectorLifecycleTests.cs @@ -63,13 +63,23 @@ public void Database_StartsInitialConnectAwayFromCallerSynchronizationContext() } [Fact] - public void IsConnected_False_BeforeConnect_DoesNotTriggerConnect() + public void IsConnected_False_OnceDisposed_EvenBeforeConnect() + { + var connector = NewConnector(new SequenceFactory()); + + connector.Dispose(); + + connector.IsConnected.Should().BeFalse("every command now fails, so none should go to the store"); + } + + [Fact] + public void IsConnected_True_BeforeConnect_DoesNotTriggerConnect() { var factory = new SequenceFactory(); var connector = NewConnector(factory); - connector.IsConnected.Should().BeFalse(); + connector.IsConnected.Should().BeTrue("not created yet is no outage: a monitored cache must still send its first command, which connects"); connector.GetEndPoints().Should().BeEmpty(); factory.CreateCount.Should().Be(0); connector.Dispose(); @@ -211,7 +221,7 @@ public async Task Dispose_DoesNotReportTheReconnectItCancelled() } [Fact] - public async Task IsConnected_False_WhileInitialConnectInFlight() + public async Task IsConnected_True_WhileInitialConnectInFlight() { var multiplexer = Substitute.For(); multiplexer.IsConnected.Returns(true); @@ -221,7 +231,7 @@ public async Task IsConnected_False_WhileInitialConnectInFlight() var warmUp = connector.ConnectAsync(TestContext.Current.CancellationToken); - connector.IsConnected.Should().BeFalse(); + connector.IsConnected.Should().BeTrue("a command sent now must wait for the pending connect rather than be refused"); connector.GetEndPoints().Should().BeEmpty(); gate.SetResult(); @@ -316,6 +326,7 @@ public async Task GetConnectionTask_SelfHeals_AfterFaultedConnect() var connector = NewConnector(factory); await Assert.ThrowsAnyAsync(async () => await connector.ConnectAsync(TestContext.Current.CancellationToken)); + connector.IsConnected.Should().BeTrue("a faulted first connect must let the next command through, or nothing would ever retry it"); await connector.ConnectAsync(TestContext.Current.CancellationToken); diff --git a/tests/UiPath.Caching.Tests/SpanKeyReadTests.cs b/tests/UiPath.Caching.Tests/SpanKeyReadTests.cs index 69a96b9..4feee16 100644 --- a/tests/UiPath.Caching.Tests/SpanKeyReadTests.cs +++ b/tests/UiPath.Caching.Tests/SpanKeyReadTests.cs @@ -269,7 +269,7 @@ public async Task A_span_read_does_not_serve_a_local_hit_when_the_connection_dro state.IsConnected.Returns(false); state.OnConnectionFailed += Raise.Event(state, EventArgs.Empty); }); - using var cache = InMemoryMultilayer.Cache(new InMemoryCacheOptions { CacheKeyStrategy = strategy }, inner, connectionMonitor: true); + using var cache = InMemoryMultilayer.Cache(new InMemoryCacheOptions { CacheKeyStrategy = strategy, UseLocalOnlyWhenDisconnected = false }, inner, connectionMonitor: true); (await cache.SetAsync("user:42", "v", policy: null, Ct)).Should().BeTrue(); SpanReads.Read(cache, "user:42", Ct).Should().BeNull("the key path removes a local entry once the tier is disconnected, and the span path must reach that decision too"); @@ -287,7 +287,7 @@ public async Task A_hash_span_read_does_not_serve_a_local_hit_when_the_connectio state.IsConnected.Returns(false); state.OnConnectionFailed += Raise.Event(state, EventArgs.Empty); }); - using var cache = InMemoryMultilayer.HashCache(new InMemoryCacheOptions { CacheKeyStrategy = strategy }, inner, connectionMonitor: true); + using var cache = InMemoryMultilayer.HashCache(new InMemoryCacheOptions { CacheKeyStrategy = strategy, UseLocalOnlyWhenDisconnected = false }, inner, connectionMonitor: true); (await cache.SetAsync("user:42", new Dictionary { ["f"] = "v" }, policy: null, Ct)).Should().BeTrue(); SpanReads.ReadItem(cache, "user:42", "f", Ct).Should().BeNull("the key path removes a local entry once the tier is disconnected, and the span path must reach that decision too");