From 306d19f4202ec96078b48cd85d4e1c2782915bbf Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 8 Jul 2026 12:50:09 -0700 Subject: [PATCH 01/47] Add opt-in blob payload auto-purge job to AzureBlobPayloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large orchestration payloads are externalized to Azure Blob Storage as `blob:v1::` tokens. The DTS backend stores those tokens but cannot delete the backing blobs (it has no storage credentials) — only this SDK can. This adds an opt-in, whole-scheduler singleton durable entity + orchestration job (mirroring src/ExportHistory) that drains payload rows the backend has soft-deleted and deletes their blobs, then acks so the backend can hard-delete the rows. Design: - PayloadStore.DeleteAsync is virtual (default throws NotSupportedException so it is non-breaking for existing external subclasses); BlobPayloadStore overrides it to decode the token and call DeleteIfExistsAsync (idempotent). - BlobPurgeJob (TaskEntity singleton): Create is a no-op when already Active so racing client processes don't disturb the running job; Run starts a fixed-id orchestrator. - BlobPurgeJobOrchestrator (perpetual): fetch a batch of tombstones, delete the blobs with capped parallelism, ack the successful deletions (failed tokens stay tombstoned to retry), idle on a timer when empty, ContinueAsNew periodically. - ExecuteBlobPurgeJobOperationOrchestrator bridges client -> entity. - Two new unary RPCs on TaskHubSidecarService: GetTombstonedPayloads / AckPurgedPayloads (authoritative proto follow-up: microsoft/durabletask-protobuf#76). - LargePayloadStorageOptions gains AutoPurge (opt-in, default false) and PayloadPurgeBatchSize (default 500). - Client-side BlobPurgeJobStarter (IHostedService) ensures the singleton job when AutoPurge is enabled, without blocking host startup. Worker always registers the entity/orchestrators/activities so a client-enabled job has something to run. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Microsoft.DurableTask.sln | 17 +- src/Client/Core/DurableTaskClient.cs | 24 +++ src/Client/Core/PayloadPurgeAckDto.cs | 13 ++ src/Client/Core/TombstonedPayloadDto.cs | 15 ++ src/Client/Grpc/GrpcDurableTaskClient.cs | 43 ++++++ .../Activities/AckPurgedPayloadsActivity.cs | 36 +++++ .../Activities/DeleteExternalBlobActivity.cs | 42 +++++ .../GetTombstonedPayloadsActivity.cs | 32 ++++ .../AutoPurge/Client/BlobPurgeJobStarter.cs | 135 ++++++++++++++++ .../AutoPurge/Constants/BlobPurgeConstants.cs | 28 ++++ .../AutoPurge/Entity/BlobPurgeJob.cs | 84 ++++++++++ .../AzureBlobPayloads/AutoPurge/Logs.cs | 39 +++++ .../Models/BlobPurgeJobCreationOptions.cs | 12 ++ .../AutoPurge/Models/BlobPurgeJobState.cs | 40 +++++ .../AutoPurge/Models/BlobPurgeJobStatus.cs | 26 ++++ .../BlobPurgeJobOrchestrator.cs | 145 ++++++++++++++++++ ...xecuteBlobPurgeJobOperationOrchestrator.cs | 30 ++++ ...ientBuilderExtensions.AzureBlobPayloads.cs | 38 +++++ ...rkerBuilderExtensions.AzureBlobPayloads.cs | 16 +- .../Options/LargePayloadStorageOptions.cs | 14 ++ .../PayloadStore/BlobPayloadStore.cs | 31 ++++ .../PayloadStore/PayloadStore.cs | 16 ++ src/Grpc/orchestrator_service.proto | 47 ++++++ .../AutoPurge/BlobPurgeJobTests.cs | 100 ++++++++++++ .../AzureBlobPayloads.Tests.csproj | 24 +++ .../LargePayloadStorageOptionsTests.cs | 21 +++ .../PayloadStore/BlobPayloadStoreTests.cs | 101 ++++++++++++ 27 files changed, 1166 insertions(+), 3 deletions(-) create mode 100644 src/Client/Core/PayloadPurgeAckDto.cs create mode 100644 src/Client/Core/TombstonedPayloadDto.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/ExecuteBlobPurgeJobOperationOrchestrator.cs create mode 100644 test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs create mode 100644 test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj create mode 100644 test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs create mode 100644 test/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreTests.cs diff --git a/Microsoft.DurableTask.sln b/Microsoft.DurableTask.sln index 3c6d4539..bf471507 100644 --- a/Microsoft.DurableTask.sln +++ b/Microsoft.DurableTask.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.3.32901.215 @@ -145,6 +145,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AzureManaged", "AzureManage EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Grpc", "Grpc", "{3B8F957E-7773-4C0C-ACD7-91A1591D9312}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureBlobPayloads.Tests", "test\AzureBlobPayloads.Tests\AzureBlobPayloads.Tests.csproj", "{531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -839,6 +841,18 @@ Global {C1995163-1DCE-405D-BE82-8B4B2584893E}.Release|x64.Build.0 = Release|Any CPU {C1995163-1DCE-405D-BE82-8B4B2584893E}.Release|x86.ActiveCfg = Release|Any CPU {C1995163-1DCE-405D-BE82-8B4B2584893E}.Release|x86.Build.0 = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|x64.ActiveCfg = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|x64.Build.0 = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|x86.ActiveCfg = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|x86.Build.0 = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|Any CPU.Build.0 = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|x64.ActiveCfg = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|x64.Build.0 = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|x86.ActiveCfg = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -911,6 +925,7 @@ Global {C1995163-1DCE-405D-BE82-8B4B2584893E} = {9686B8F9-2644-6C9B-E567-55B0471E4584} {53193780-CD18-2643-6953-C26F59EAEDF5} = {5B448FF6-EC42-491D-A22E-1DC8B618E6D5} {3B8F957E-7773-4C0C-ACD7-91A1591D9312} = {5B448FF6-EC42-491D-A22E-1DC8B618E6D5} + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5} = {E5637F81-2FB9-4CD7-900D-455363B142A7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {AB41CB55-35EA-4986-A522-387AB3402E71} diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index 03303800..208e6441 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -549,6 +549,30 @@ public virtual Task> ListInstanceIdsAsync( $"{this.GetType()} does not support listing orchestration instance IDs filtered by completed time."); } + /// + /// Gets a batch of tombstoned (soft-deleted) externalized payloads whose backing blobs should be deleted + /// by a credentialed caller before the backend hard-deletes the rows. + /// + /// The maximum number of tombstoned payloads to request. + /// The cancellation token. + /// The batch of tombstoned payloads whose blobs should be deleted. + /// Thrown if this implementation does not support the operation. + public virtual Task> GetTombstonedPayloadsAsync( + int limit, CancellationToken cancellation = default) + => throw new NotSupportedException($"{this.GetType()} does not support retrieving tombstoned payloads."); + + /// + /// Acknowledges tombstoned payloads whose backing blobs have been deleted so the backend can hard-delete + /// the corresponding rows. + /// + /// The payloads whose blobs have been deleted. + /// The cancellation token. + /// A task that completes when the acknowledgement has been sent. + /// Thrown if this implementation does not support the operation. + public virtual Task AckPurgedPayloadsAsync( + IEnumerable acks, CancellationToken cancellation = default) + => throw new NotSupportedException($"{this.GetType()} does not support acknowledging purged payloads."); + // TODO: Create task hub // TODO: Delete task hub diff --git a/src/Client/Core/PayloadPurgeAckDto.cs b/src/Client/Core/PayloadPurgeAckDto.cs new file mode 100644 index 00000000..01db0f21 --- /dev/null +++ b/src/Client/Core/PayloadPurgeAckDto.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable acknowledgement that the worker has deleted the blob for a tombstoned payload, so the +/// backend can hard-delete the soft-deleted row. Mirrors the PayloadPurgeAck protobuf message. +/// +/// The backend partition that owns the payload row. +/// The orchestration instance key the payload belongs to. +/// The backend identifier of the soft-deleted payload row. +public sealed record PayloadPurgeAckDto(int PartitionId, long InstanceKey, long PayloadId); diff --git a/src/Client/Core/TombstonedPayloadDto.cs b/src/Client/Core/TombstonedPayloadDto.cs new file mode 100644 index 00000000..69308d47 --- /dev/null +++ b/src/Client/Core/TombstonedPayloadDto.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable representation of a tombstoned payload the backend has soft-deleted and whose blob the +/// worker should delete. Mirrors the TombstonedPayload protobuf message but is safe to pass through +/// the orchestration/activity boundary. +/// +/// The backend partition that owns the payload row. +/// The orchestration instance key the payload belongs to. +/// The backend identifier of the soft-deleted payload row. +/// The externalized payload token whose backing blob should be deleted. +public sealed record TombstonedPayloadDto(int PartitionId, long InstanceKey, long PayloadId, string Token); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 23350d4c..822f7238 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -624,6 +624,49 @@ public override async Task> GetOrchestrationHistoryAsync( } } + /// + public override async Task> GetTombstonedPayloadsAsync( + int limit, CancellationToken cancellation = default) + { + P.GetTombstonedPayloadsResponse response = await this.sidecarClient.GetTombstonedPayloadsAsync( + new P.GetTombstonedPayloadsRequest { Limit = limit }, + cancellationToken: cancellation); + + List result = new(response.Payloads.Count); + foreach (P.TombstonedPayload payload in response.Payloads) + { + result.Add(new TombstonedPayloadDto( + payload.PartitionId, payload.InstanceKey, payload.PayloadId, payload.Token)); + } + + return result; + } + + /// + public override async Task AckPurgedPayloadsAsync( + IEnumerable acks, CancellationToken cancellation = default) + { + Check.NotNull(acks); + + P.AckPurgedPayloadsRequest request = new(); + foreach (PayloadPurgeAckDto ack in acks) + { + request.Acks.Add(new P.PayloadPurgeAck + { + PartitionId = ack.PartitionId, + InstanceKey = ack.InstanceKey, + PayloadId = ack.PayloadId, + }); + } + + if (request.Acks.Count == 0) + { + return; + } + + await this.sidecarClient.AckPurgedPayloadsAsync(request, cancellationToken: cancellation); + } + static AsyncDisposable GetCallInvoker(GrpcDurableTaskClientOptions options, ILogger logger, out CallInvoker callInvoker) { Func>? recreator = options.Internal.ChannelRecreator; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs new file mode 100644 index 00000000..637fa5ec --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that acknowledges to the backend the payloads whose blobs the worker has deleted, so the backend +/// can hard-delete the soft-deleted rows. +/// +/// The Durable Task client used to acknowledge purged payloads to the backend. +/// The logger instance. +[DurableTask] +public class AckPurgedPayloadsActivity( + DurableTaskClient client, + ILogger logger) + : TaskActivity, object?> +{ + readonly DurableTaskClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task RunAsync(TaskActivityContext context, List input) + { + if (input is null || input.Count == 0) + { + return null; + } + + await this.client.AckPurgedPayloadsAsync(input, CancellationToken.None); + this.logger.BlobPurgeAckedPayloads(input.Count); + return null; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs new file mode 100644 index 00000000..0018cb7c --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that deletes a single externalized payload blob given its token. Deletion is idempotent, so +/// re-delivered tokens and concurrent workers are safe. On failure the payload is left tombstoned so a later +/// purge cycle can retry it. +/// +/// The payload store used to delete blobs. +/// The logger instance. +[DurableTask] +public class DeleteExternalBlobActivity( + PayloadStore store, + ILogger logger) + : TaskActivity +{ + readonly PayloadStore store = Check.NotNull(store); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task RunAsync(TaskActivityContext context, string input) + { + Check.NotNullOrEmpty(input, nameof(input)); + + try + { + await this.store.DeleteAsync(input, CancellationToken.None); + return true; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + // Leave the payload tombstoned so the backend re-streams it on a later cycle; a single bad token + // must not fail the whole batch. + this.logger.BlobPurgeDeleteFailed(ex, input); + return false; + } + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs new file mode 100644 index 00000000..6086efe2 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that fetches a batch of tombstoned payloads from the backend for the auto-purge job to delete. +/// +/// The Durable Task client used to query the backend for tombstoned payloads. +/// The logger instance. +[DurableTask] +public class GetTombstonedPayloadsActivity( + DurableTaskClient client, + ILogger logger) + : TaskActivity> +{ + readonly DurableTaskClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task> RunAsync(TaskActivityContext context, int input) + { + int limit = input > 0 ? input : 500; + List payloads = + await this.client.GetTombstonedPayloadsAsync(limit, CancellationToken.None); + this.logger.BlobPurgeFetchedTombstones(payloads.Count); + return payloads; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs new file mode 100644 index 00000000..9652c1e3 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Client-side hosted service that ensures the singleton blob payload auto-purge job exists when auto-purge +/// is enabled via . It never blocks host startup: it runs +/// on a background task and retries until the backend is reachable. The job is a whole-scheduler singleton, +/// so racing client processes simply no-op. +/// +sealed class BlobPurgeJobStarter : IHostedService +{ + static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); + + readonly DurableTaskClient client; + readonly IOptionsMonitor options; + readonly string builderName; + readonly ILogger logger; + readonly EntityInstanceId entityId = new(nameof(BlobPurgeJob), BlobPurgeConstants.JobId); + + CancellationTokenSource? cts; + Task? ensureTask; + + public BlobPurgeJobStarter( + DurableTaskClient client, + IOptionsMonitor options, + string builderName, + ILogger logger) + { + this.client = Check.NotNull(client); + this.options = Check.NotNull(options); + this.builderName = Check.NotNull(builderName); + this.logger = Check.NotNull(logger); + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + LargePayloadStorageOptions opts = this.options.Get(this.builderName); + if (!opts.AutoPurge) + { + this.logger.BlobPurgeDisabled(); + return Task.CompletedTask; + } + + int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : 500; + + // Do not block host startup; ensure the job on a background task with basic retry until the backend + // is reachable. + this.cts = new CancellationTokenSource(); + this.ensureTask = Task.Run(() => this.EnsureJobAsync(batchSize, this.cts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + this.cts?.Cancel(); + + Task? pending = this.ensureTask; + if (pending is not null) + { + // The ensure loop observes cancellation and returns promptly; swallow any faulted/cancelled result. + await Task.WhenAny(pending, Task.Delay(Timeout.Infinite, cancellationToken)).ConfigureAwait(false); + } + } + + async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + if (await this.IsJobActiveAsync(cancellationToken)) + { + return; + } + + BlobPurgeJobOperationRequest request = new( + this.entityId, + nameof(BlobPurgeJob.Create), + new BlobPurgeJobCreationOptions(batchSize)); + + await this.client.ScheduleNewOrchestrationInstanceAsync( + new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), + request, + cancellationToken); + + this.logger.BlobPurgeJobEnsured(); + return; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + this.logger.BlobPurgeStarterRetry(ex); + try + { + await Task.Delay(RetryDelay, cancellationToken); + } + catch (OperationCanceledException) + { + return; + } + } + } + } + + async Task IsJobActiveAsync(CancellationToken cancellationToken) + { + try + { + EntityMetadata? metadata = + await this.client.Entities.GetEntityAsync( + this.entityId, cancellation: cancellationToken); + return metadata is not null && metadata.State.Status == BlobPurgeJobStatus.Active; + } + catch (NotSupportedException) + { + // The entity-query API is unavailable on this client; fall back to scheduling the idempotent + // Create, which no-ops if the job is already active. + return false; + } + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs new file mode 100644 index 00000000..eafee991 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Constants used throughout the blob payload auto-purge functionality. +/// +static class BlobPurgeConstants +{ + /// + /// The fixed, process-global job ID for the singleton blob payload auto-purge job. A single job drains + /// tombstoned payloads for the whole scheduler, so the ID is hard-coded rather than caller-supplied. + /// + public const string JobId = "__dt_blob_payload_autopurge__"; + + /// + /// The prefix used for generating blob purge job orchestrator instance IDs. Format: "BlobPurgeJob-{jobId}". + /// + public const string OrchestratorInstanceIdPrefix = "BlobPurgeJob-"; + + /// + /// Generates an orchestrator instance ID for a given blob purge job ID. + /// + /// The blob purge job ID. + /// The orchestrator instance ID. + public static string GetOrchestratorInstanceId(string jobId) => $"{OrchestratorInstanceIdPrefix}{jobId}"; +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs new file mode 100644 index 00000000..8f08b867 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Durable entity that manages the lifecycle of the singleton blob payload auto-purge job. +/// +/// The logger instance. +class BlobPurgeJob(ILogger logger) : TaskEntity +{ + /// + /// Creates (or reactivates) the auto-purge job. Because the job is a whole-scheduler singleton, this is + /// intentionally a no-op when the job is already so that extra + /// client processes racing to create it do not disturb the running job. + /// + /// The entity context. + /// The job creation options. + public void Create(TaskEntityContext context, BlobPurgeJobCreationOptions creationOptions) + { + Check.NotNull(creationOptions, nameof(creationOptions)); + + if (this.State.Status == BlobPurgeJobStatus.Active) + { + logger.BlobPurgeJobAlreadyRunning(context.Id.Key); + return; + } + + this.State.Status = BlobPurgeJobStatus.Active; + this.State.PurgeBatchSize = creationOptions.PurgeBatchSize > 0 ? creationOptions.PurgeBatchSize : 500; + this.State.CreatedAt ??= DateTimeOffset.UtcNow; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + this.State.LastError = null; + + logger.BlobPurgeJobCreated(context.Id.Key); + + // Signal Run to start the perpetual purge orchestrator. + context.SignalEntity(context.Id, nameof(this.Run)); + } + + /// + /// Starts the purge orchestrator if the job is active. Uses a fixed orchestrator instance ID so only one + /// orchestrator ever runs for the singleton job. + /// + /// The entity context. + public void Run(TaskEntityContext context) + { + if (this.State.Status != BlobPurgeJobStatus.Active) + { + return; + } + + string instanceId = BlobPurgeConstants.GetOrchestratorInstanceId(context.Id.Key); + StartOrchestrationOptions startOrchestrationOptions = new(instanceId); + + context.ScheduleNewOrchestration( + new TaskName(nameof(BlobPurgeJobOrchestrator)), + new BlobPurgeJobRunRequest(context.Id, this.State.PurgeBatchSize), + startOrchestrationOptions); + + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } + + /// + /// Records progress after a purge cycle completes. + /// + /// The entity context. + /// The number of blobs purged in the cycle. + public void RecordPurged(TaskEntityContext context, long purgedCount) + { + this.State.PurgedCount += purgedCount; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } + + /// + /// Gets the current state of the auto-purge job. + /// + /// The entity context. + /// The current job state. + public BlobPurgeJobState Get(TaskEntityContext context) => this.State; +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs new file mode 100644 index 00000000..6e4ebed8 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Log messages for the Azure Blob externalized-payload auto-purge job. +/// +static partial class Logs +{ + [LoggerMessage(EventId = 810, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' created.")] + public static partial void BlobPurgeJobCreated(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 811, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' is already running; ignoring the create request.")] + public static partial void BlobPurgeJobAlreadyRunning(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] + public static partial void BlobPurgeJobOrchestratorStopping(this ILogger logger, string? jobId, string status); + + [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Failed to delete externalized payload blob for token '{token}'; leaving it tombstoned for a later purge cycle.")] + public static partial void BlobPurgeDeleteFailed(this ILogger logger, Exception exception, string token); + + [LoggerMessage(EventId = 814, Level = LogLevel.Debug, Message = "Blob payload auto-purge fetched {count} tombstoned payload(s) from the backend.")] + public static partial void BlobPurgeFetchedTombstones(this ILogger logger, int count); + + [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge acknowledged {count} purged payload(s) to the backend.")] + public static partial void BlobPurgeAckedPayloads(this ILogger logger, int count); + + [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge is disabled; the singleton purge job will not be started.")] + public static partial void BlobPurgeDisabled(this ILogger logger); + + [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] + public static partial void BlobPurgeJobEnsured(this ILogger logger); + + [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] + public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs new file mode 100644 index 00000000..e49e423c --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Options used to create the singleton blob payload auto-purge job. +/// +/// +/// The maximum number of tombstoned payloads to request from the backend per cycle. +/// +public sealed record BlobPurgeJobCreationOptions(int PurgeBatchSize); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs new file mode 100644 index 00000000..8bd4cdd5 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// State for the singleton blob payload auto-purge job, stored in the entity. +/// +public sealed class BlobPurgeJobState +{ + /// + /// Gets or sets the current status of the auto-purge job. + /// + public BlobPurgeJobStatus Status { get; set; } + + /// + /// Gets or sets the time when the job was first created. + /// + public DateTimeOffset? CreatedAt { get; set; } + + /// + /// Gets or sets the time when the job state was last modified. + /// + public DateTimeOffset? LastModifiedAt { get; set; } + + /// + /// Gets or sets the total number of payload blobs the job has purged. + /// + public long PurgedCount { get; set; } + + /// + /// Gets or sets the last error message, if any. + /// + public string? LastError { get; set; } + + /// + /// Gets or sets the maximum number of tombstoned payloads requested from the backend per cycle. + /// + public int PurgeBatchSize { get; set; } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs new file mode 100644 index 00000000..917b8d1b --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Represents the current status of the singleton blob payload auto-purge job. +/// +public enum BlobPurgeJobStatus +{ + /// + /// The job is not running. This is the default status of a freshly initialized entity, so it is kept + /// as the zero value to avoid a brand-new entity accidentally appearing active. + /// + Stopped, + + /// + /// The job is active and draining tombstoned payloads from the backend. + /// + Active, + + /// + /// The job has failed. + /// + Failed, +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs new file mode 100644 index 00000000..2b7ba6f2 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Orchestrator input describing the purge job to run. +/// +/// The entity ID of the owning . +/// The maximum number of tombstoned payloads to request per cycle. +/// The number of cycles processed since the last continue-as-new. +public sealed record BlobPurgeJobRunRequest( + EntityInstanceId JobEntityId, int PurgeBatchSize, int ProcessedCycles = 0); + +/// +/// Perpetual orchestrator that drains tombstoned payloads from the backend, deletes their blobs with capped +/// parallelism, and acknowledges the successful deletions so the backend can hard-delete the rows. It idles +/// on a timer when there is nothing to purge and continues-as-new periodically to keep its history small. +/// +[DurableTask] +public class BlobPurgeJobOrchestrator : TaskOrchestrator +{ + const int ContinueAsNewFrequency = 5; + const int MaxParallelDeletes = 32; + const int DefaultPurgeBatchSize = 500; + static readonly TimeSpan IdleDelay = TimeSpan.FromMinutes(1); + + // Retry policy for the purge activities: 3 attempts with exponential backoff (15s, 30s, capped at 60s). + static readonly RetryPolicy PurgeActivityRetryPolicy = new( + maxNumberOfAttempts: 3, + firstRetryInterval: TimeSpan.FromSeconds(15), + backoffCoefficient: 2.0, + maxRetryInterval: TimeSpan.FromSeconds(60)); + + /// + public override async Task RunAsync(TaskOrchestrationContext context, BlobPurgeJobRunRequest input) + { + ILogger logger = context.CreateReplaySafeLogger(); + string jobId = input.JobEntityId.Key; + + int batchSize = input.PurgeBatchSize > 0 ? input.PurgeBatchSize : DefaultPurgeBatchSize; + int processedCycles = input.ProcessedCycles; + + while (true) + { + processedCycles++; + if (processedCycles > ContinueAsNewFrequency) + { + context.ContinueAsNew(new BlobPurgeJobRunRequest(input.JobEntityId, batchSize, ProcessedCycles: 0)); + return null!; + } + + // Stop cleanly if the job has been stopped or removed. + BlobPurgeJobState? state = await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.Get), null); + + if (state is null || state.Status != BlobPurgeJobStatus.Active) + { + logger.BlobPurgeJobOrchestratorStopping(jobId, state?.Status.ToString() ?? "null"); + return null; + } + + List tombstones = await context.CallActivityAsync>( + nameof(GetTombstonedPayloadsActivity), + batchSize, + new TaskOptions(PurgeActivityRetryPolicy)); + + if (tombstones is null || tombstones.Count == 0) + { + // Nothing to purge right now: block on a timer (push-free idle) then check again. + await context.CreateTimer(IdleDelay, default); + continue; + } + + List deleted = await this.DeleteBatchAsync(context, tombstones); + + if (deleted.Count > 0) + { + await context.CallActivityAsync( + nameof(AckPurgedPayloadsActivity), + deleted, + new TaskOptions(PurgeActivityRetryPolicy)); + + await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)deleted.Count); + } + } + } + + async Task> DeleteBatchAsync( + TaskOrchestrationContext context, List tombstones) + { + List deleted = new(tombstones.Count); + List> tasks = new(); + + foreach (TombstonedPayloadDto tombstone in tombstones) + { + tasks.Add(this.DeleteOneAsync(context, tombstone)); + + if (tasks.Count >= MaxParallelDeletes) + { + await DrainAsync(tasks, deleted); + tasks.Clear(); + } + } + + if (tasks.Count > 0) + { + await DrainAsync(tasks, deleted); + } + + return deleted; + } + + static async Task DrainAsync(List> tasks, List deleted) + { + DeleteOutcome[] outcomes = await Task.WhenAll(tasks); + foreach (DeleteOutcome outcome in outcomes) + { + // Only acknowledge blobs that were actually deleted; failed tokens stay tombstoned to retry. + if (outcome.Deleted) + { + deleted.Add(outcome.Ack); + } + } + } + + async Task DeleteOneAsync(TaskOrchestrationContext context, TombstonedPayloadDto tombstone) + { + bool deleted = await context.CallActivityAsync( + nameof(DeleteExternalBlobActivity), + tombstone.Token, + new TaskOptions(PurgeActivityRetryPolicy)); + + return new DeleteOutcome( + deleted, + new PayloadPurgeAckDto(tombstone.PartitionId, tombstone.InstanceKey, tombstone.PayloadId)); + } + + readonly record struct DeleteOutcome(bool Deleted, PayloadPurgeAckDto Ack); +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/ExecuteBlobPurgeJobOperationOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/ExecuteBlobPurgeJobOperationOrchestrator.cs new file mode 100644 index 00000000..e8f0c66b --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/ExecuteBlobPurgeJobOperationOrchestrator.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Orchestrator that executes a single operation on a blob purge job entity and returns the result. Used as +/// a client-to-entity bridge so clients can drive the entity through the orchestration surface. +/// +[DurableTask] +public class ExecuteBlobPurgeJobOperationOrchestrator + : TaskOrchestrator +{ + /// + public override async Task RunAsync( + TaskOrchestrationContext context, BlobPurgeJobOperationRequest input) + { + return await context.Entities.CallEntityAsync(input.EntityId, input.OperationName, input.Input); + } +} + +/// +/// Request for executing a blob purge job entity operation. +/// +/// The ID of the entity to execute the operation on. +/// The name of the operation to execute. +/// Optional input for the operation. +public record BlobPurgeJobOperationRequest(EntityInstanceId EntityId, string OperationName, object? Input = null); diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 0817bcea..2557c6da 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -2,11 +2,14 @@ // Licensed under the MIT License. using Grpc.Core.Interceptors; +using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Client.Grpc; using Microsoft.DurableTask.Converters; using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Microsoft.DurableTask; @@ -16,6 +19,29 @@ namespace Microsoft.DurableTask; /// public static class DurableTaskClientBuilderExtensionsAzureBlobPayloads { + /// + /// Enables externalized payload storage using Azure Blob Storage for the specified client builder. + /// + /// The builder to configure. + /// The callback to configure the storage options. + /// The original builder, for call chaining. + public static IDurableTaskClientBuilder UseExternalizedPayloads( + this IDurableTaskClientBuilder builder, + Action configure) + { + Check.NotNull(builder); + Check.NotNull(configure); + + builder.Services.Configure(builder.Name, configure); + builder.Services.AddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + + return UseExternalizedPayloadsCore(builder); + } + /// /// Enables externalized payload storage using a pre-configured shared payload store. /// This overload helps ensure client and worker use the same configuration. @@ -56,6 +82,18 @@ static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientB } }); + RegisterBlobPurgeJobStarter(builder); + return builder; } + + static void RegisterBlobPurgeJobStarter(IDurableTaskClientBuilder builder) + { + string builderName = builder.Name; + builder.Services.AddSingleton(sp => new BlobPurgeJobStarter( + sp.GetRequiredService(), + sp.GetRequiredService>(), + builderName, + sp.GetRequiredService>())); + } } diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index b690d288..42ac0f0a 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -2,8 +2,7 @@ // Licensed under the MIT License. using Grpc.Core.Interceptors; -using Grpc.Net.Client; -using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; @@ -82,6 +81,19 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB opt.Capabilities.Add(P.WorkerCapability.LargePayloads); }); + // Register the entity/orchestrators/activities that run the singleton auto-purge job. These are + // ALWAYS registered (not gated on AutoPurge) so that a client-enabled job always has something to + // execute here. The purge activities fetch/ack via the injected DurableTaskClient. + builder.AddTasks(r => + { + r.AddEntity(); + r.AddOrchestrator(); + r.AddOrchestrator(); + r.AddActivity(); + r.AddActivity(); + r.AddActivity(); + }); + return builder; } } diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 6abcbdf2..f53c9e72 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -115,4 +115,18 @@ public int ThresholdBytes /// Defaults to true for reduced storage and bandwidth. /// public bool CompressionEnabled { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the client should start the singleton blob payload auto-purge + /// job. When enabled, the job periodically drains payload rows the backend has soft-deleted and deletes + /// the corresponding blobs from customer storage (the backend has no storage credentials of its own). + /// Defaults to false (opt-in). + /// + public bool AutoPurge { get; set; } + + /// + /// Gets or sets the maximum number of tombstoned payloads the auto-purge job requests from the backend + /// per cycle. Defaults to 500. Values less than or equal to zero are treated as the default. + /// + public int PayloadPurgeBatchSize { get; set; } = 500; } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index e01d2e74..1934c5d4 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -67,6 +67,18 @@ public BlobPayloadStore(LargePayloadStorageOptions options) this.containerClient = serviceClient.GetBlobContainerClient(options.ContainerName); } + /// + /// Initializes a new instance of the class using a caller-supplied + /// container client. Intended for unit testing so a mocked can be injected. + /// + /// The blob container client to use. + /// The options for the blob payload store. + internal BlobPayloadStore(BlobContainerClient containerClient, LargePayloadStorageOptions options) + { + this.containerClient = containerClient ?? throw new ArgumentNullException(nameof(containerClient)); + this.options = options ?? throw new ArgumentNullException(nameof(options)); + } + /// public override async Task UploadAsync(string payLoad, CancellationToken cancellationToken) { @@ -145,6 +157,25 @@ public override async Task DownloadAsync(string token, CancellationToken } } + /// + public override async Task DeleteAsync(string token, CancellationToken cancellationToken) + { + (string container, string name) = DecodeToken(token); + if (!string.Equals(container, this.containerClient.Name, StringComparison.Ordinal)) + { + throw new ArgumentException("Token container does not match configured container.", nameof(token)); + } + + BlobClient blob = this.containerClient.GetBlobClient(name); + + // Idempotent by design: DeleteIfExistsAsync returns false (rather than throwing) when the blob is + // already gone, so re-delivered tombstones and concurrent purges from multiple worker replicas are safe. + await blob.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + conditions: null, + cancellationToken); + } + /// public override bool IsKnownPayloadToken(string value) { diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs index b0fe6f80..0ae5ccb6 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs @@ -24,6 +24,22 @@ public abstract class PayloadStore /// Payload string. public abstract Task DownloadAsync(string token, CancellationToken cancellationToken); + /// + /// Deletes the payload referenced by the token. Implementations that support deletion must be + /// idempotent: deleting a payload that no longer exists is a no-op and must not throw. + /// + /// + /// The default implementation throws . Stores that externalize + /// payloads to deletable storage (for example Azure Blob Storage) should override it. It is declared + /// virtual rather than abstract so that adding it does not break existing external subclasses. + /// + /// The opaque reference token. + /// Cancellation token. + /// A task that completes when the payload has been deleted (or was already absent). + public virtual Task DeleteAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException( + $"This {nameof(PayloadStore)} implementation does not support deleting payloads."); + /// /// Returns true if the specified value appears to be a token understood by this store. /// Implementations should not throw for unknown tokens. diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 3d9194ac..af065941 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -823,6 +823,53 @@ service TaskHubSidecarService { // "Skip" graceful termination of orchestrations by immediately changing their status in storage to "terminated". // Note that a maximum of 500 orchestrations can be terminated at a time using this method. rpc SkipGracefulOrchestrationTerminations(SkipGracefulOrchestrationTerminationsRequest) returns (SkipGracefulOrchestrationTerminationsResponse); + + // Returns a batch of blob-externalized payload tombstones the backend has soft-deleted so the worker + // can delete the corresponding blobs from customer storage (the backend has no storage credentials of + // its own). The worker deletes each blob and then calls AckPurgedPayloads so the backend can + // hard-delete the soft-deleted rows. This is an opt-in, worker-driven pull model. + rpc GetTombstonedPayloads(GetTombstonedPayloadsRequest) returns (GetTombstonedPayloadsResponse); + + // Acknowledges that the worker has deleted the blobs for the identified payloads so the backend can + // hard-delete the soft-deleted rows. + rpc AckPurgedPayloads(AckPurgedPayloadsRequest) returns (AckPurgedPayloadsResponse); +} + +// Server -> client. Identifies a blob-externalized payload that the backend has soft-deleted and whose +// blob the worker should delete from customer storage. +message TombstonedPayload { + int32 partitionId = 1; + int64 instanceKey = 2; + int64 payloadId = 3; + // The externalized payload token (e.g. "blob:v1::") whose backing blob should be deleted. + string token = 4; +} + +// Client -> server. Acknowledges that the worker has deleted the blob for the identified payload so the +// backend can hard-delete the soft-deleted row. +message PayloadPurgeAck { + int32 partitionId = 1; + int64 instanceKey = 2; + int64 payloadId = 3; +} + +// Client -> server. Requests up to `limit` tombstoned payloads for the worker to delete. +message GetTombstonedPayloadsRequest { + int32 limit = 1; +} + +// Server -> client. Carries the batch of tombstoned payloads for the worker to delete. +message GetTombstonedPayloadsResponse { + repeated TombstonedPayload payloads = 1; +} + +// Client -> server. Acknowledges a batch of payloads whose blobs the worker has deleted. +message AckPurgedPayloadsRequest { + repeated PayloadPurgeAck acks = 1; +} + +// Server -> client. Empty response acknowledging the acks were recorded. +message AckPurgedPayloadsResponse { } message GetWorkItemsRequest { diff --git a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs new file mode 100644 index 00000000..126afdf8 --- /dev/null +++ b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Entities.Tests; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class BlobPurgeJobTests +{ + readonly BlobPurgeJob job = new(new TestLogger()); + + [Fact] + public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() + { + // Arrange + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(null), + new BlobPurgeJobCreationOptions(250)); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(250); + state.CreatedAt.Should().NotBeNull(); + state.LastModifiedAt.Should().NotBeNull(); + } + + [Fact] + public async Task Create_WhenAlreadyActive_IsNoOp() + { + // Arrange + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 100, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + new BlobPurgeJobCreationOptions(999)); + + // Act + await this.job.RunAsync(operation); + + // Assert - status stays Active and the original batch size is retained, proving the create no-op'd. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(100); + } + + [Fact] + public async Task Create_WithNonPositiveBatchSize_FallsBackToDefault() + { + // Arrange + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(null), + new BlobPurgeJobCreationOptions(0)); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.PurgeBatchSize.Should().Be(500); + } + + [Fact] + public async Task Get_ReturnsCurrentState() + { + // Arrange + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 42, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Get), + new TestEntityState(existing), + null); + + // Act + object? result = await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType(result); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(42); + } +} diff --git a/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj b/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj new file mode 100644 index 00000000..77078cbb --- /dev/null +++ b/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + false + true + + Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests + Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests + + + + + + + + + + + + diff --git a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs new file mode 100644 index 00000000..317b1e1e --- /dev/null +++ b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.Options; + +public class LargePayloadStorageOptionsTests +{ + [Fact] + public void Defaults_AutoPurgeDisabled_AndBatchSize500() + { + // Arrange & Act + LargePayloadStorageOptions options = new(); + + // Assert + options.AutoPurge.Should().BeFalse(); + options.PayloadPurgeBatchSize.Should().Be(500); + } +} diff --git a/test/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreTests.cs b/test/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreTests.cs new file mode 100644 index 00000000..e7fe1448 --- /dev/null +++ b/test/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreTests.cs @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests; + +public class BlobPayloadStoreTests +{ + const string ContainerName = "payloads"; + + static Mock CreateContainer(Mock blob, string expectedBlobName) + { + Mock container = new(); + container.Setup(c => c.Name).Returns(ContainerName); + container.Setup(c => c.GetBlobClient(expectedBlobName)).Returns(blob.Object); + return container; + } + + static Mock CreateBlob(bool existed) + { + Mock blob = new(); + blob + .Setup(b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(existed, Mock.Of())); + return blob; + } + + [Fact] + public async Task DeleteAsync_ValidToken_DeletesBackingBlobIncludingSnapshots() + { + // Arrange + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(container.Object, new LargePayloadStorageOptions()); + + // Act + await store.DeleteAsync($"blob:v1:{ContainerName}:abc123", CancellationToken.None); + + // Assert + container.Verify(c => c.GetBlobClient("abc123"), Times.Once); + blob.Verify( + b => b.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DeleteAsync_MissingBlob_IsIdempotentAndDoesNotThrow() + { + // Arrange + Mock blob = CreateBlob(existed: false); + Mock container = CreateContainer(blob, "missing"); + BlobPayloadStore store = new(container.Object, new LargePayloadStorageOptions()); + + // Act (a missing blob must be a no-op, not an error) + await store.DeleteAsync($"blob:v1:{ContainerName}:missing", CancellationToken.None); + + // Assert + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DeleteAsync_ContainerMismatch_ThrowsAndDoesNotDelete() + { + // Arrange + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(container.Object, new LargePayloadStorageOptions()); + + // Act & Assert + await Assert.ThrowsAsync( + () => store.DeleteAsync("blob:v1:other-container:abc123", CancellationToken.None)); + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Theory] + [InlineData("not-a-token")] + [InlineData("blob:v1:only-container")] + [InlineData("blob:v1::blobname")] + public async Task DeleteAsync_InvalidToken_ThrowsArgumentException(string token) + { + // Arrange + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(container.Object, new LargePayloadStorageOptions()); + + // Act & Assert + await Assert.ThrowsAsync(() => store.DeleteAsync(token, CancellationToken.None)); + } +} From 60f6637bb5983914d0c4de592bf5e30383e693a2 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 13 Jul 2026 18:39:26 -0700 Subject: [PATCH 02/47] Address PR #758 review feedback: naming, self-heal, poison-ack, starter simplification - Drop the `Dto` suffix now that the payload records are first-class public types in `Microsoft.DurableTask.Client` (`TombstonedPayload`, `PayloadPurgeAck`). - Collapse the magic `500` batch-size literal into a single `BlobPurgeConstants.DefaultBatchSize` used everywhere. - Rename `BlobPurgeJobStatus.Stopped` -> `Pending` (still the zero value) and remove the dead `Failed` member (nothing ever set it; the job self-heals). - Make the perpetual orchestrator self-heal: wrap each cycle in try/catch so a transient backend/entity/activity failure logs, backs off, and continues instead of failing the orchestration and killing the eternal loop. - Ack poison tokens: `DeleteExternalBlobActivity` now returns a three-way `BlobDeleteResult` (Deleted/Discarded/Retry). Malformed tokens are discarded and acked so the backend can clear the stuck row instead of re-streaming it forever; transient failures stay tombstoned to retry. - Replace the single-value `BlobPurgeJobCreationOptions` record with a plain `int` on `BlobPurgeJob.Create`. - Guard the client fetch RPC: `GetTombstonedPayloadsAsync` throws `ArgumentOutOfRangeException` unless `0 < limit < 1000`. - Simplify `BlobPurgeJobStarter` to a fixed-instance-id fire-once: drop the entity-active pre-check and schedule the Create bridge once with a fixed instance id, retrying only until the backend is reachable. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Client/Core/DurableTaskClient.cs | 4 +- ...yloadPurgeAckDto.cs => PayloadPurgeAck.cs} | 2 +- ...onedPayloadDto.cs => TombstonedPayload.cs} | 2 +- src/Client/Grpc/GrpcDurableTaskClient.cs | 16 ++- .../Activities/AckPurgedPayloadsActivity.cs | 4 +- .../Activities/DeleteExternalBlobActivity.cs | 23 ++-- .../GetTombstonedPayloadsActivity.cs | 8 +- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 33 +----- .../AutoPurge/Constants/BlobPurgeConstants.cs | 12 ++ .../AutoPurge/Entity/BlobPurgeJob.cs | 10 +- .../AzureBlobPayloads/AutoPurge/Logs.cs | 6 + .../AutoPurge/Models/BlobDeleteResult.cs | 27 +++++ .../Models/BlobPurgeJobCreationOptions.cs | 12 -- .../AutoPurge/Models/BlobPurgeJobStatus.cs | 11 +- .../BlobPurgeJobOrchestrator.cs | 109 ++++++++++-------- .../Options/LargePayloadStorageOptions.cs | 3 +- .../AutoPurge/BlobPurgeJobTests.cs | 6 +- 17 files changed, 160 insertions(+), 128 deletions(-) rename src/Client/Core/{PayloadPurgeAckDto.cs => PayloadPurgeAck.cs} (87%) rename src/Client/Core/{TombstonedPayloadDto.cs => TombstonedPayload.cs} (87%) create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index 208e6441..bc31915c 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -557,7 +557,7 @@ public virtual Task> ListInstanceIdsAsync( /// The cancellation token. /// The batch of tombstoned payloads whose blobs should be deleted. /// Thrown if this implementation does not support the operation. - public virtual Task> GetTombstonedPayloadsAsync( + public virtual Task> GetTombstonedPayloadsAsync( int limit, CancellationToken cancellation = default) => throw new NotSupportedException($"{this.GetType()} does not support retrieving tombstoned payloads."); @@ -570,7 +570,7 @@ public virtual Task> GetTombstonedPayloadsAsync( /// A task that completes when the acknowledgement has been sent. /// Thrown if this implementation does not support the operation. public virtual Task AckPurgedPayloadsAsync( - IEnumerable acks, CancellationToken cancellation = default) + IEnumerable acks, CancellationToken cancellation = default) => throw new NotSupportedException($"{this.GetType()} does not support acknowledging purged payloads."); // TODO: Create task hub diff --git a/src/Client/Core/PayloadPurgeAckDto.cs b/src/Client/Core/PayloadPurgeAck.cs similarity index 87% rename from src/Client/Core/PayloadPurgeAckDto.cs rename to src/Client/Core/PayloadPurgeAck.cs index 01db0f21..6e7788aa 100644 --- a/src/Client/Core/PayloadPurgeAckDto.cs +++ b/src/Client/Core/PayloadPurgeAck.cs @@ -10,4 +10,4 @@ namespace Microsoft.DurableTask.Client; /// The backend partition that owns the payload row. /// The orchestration instance key the payload belongs to. /// The backend identifier of the soft-deleted payload row. -public sealed record PayloadPurgeAckDto(int PartitionId, long InstanceKey, long PayloadId); +public sealed record PayloadPurgeAck(int PartitionId, long InstanceKey, long PayloadId); diff --git a/src/Client/Core/TombstonedPayloadDto.cs b/src/Client/Core/TombstonedPayload.cs similarity index 87% rename from src/Client/Core/TombstonedPayloadDto.cs rename to src/Client/Core/TombstonedPayload.cs index 69308d47..bb3f6896 100644 --- a/src/Client/Core/TombstonedPayloadDto.cs +++ b/src/Client/Core/TombstonedPayload.cs @@ -12,4 +12,4 @@ namespace Microsoft.DurableTask.Client; /// The orchestration instance key the payload belongs to. /// The backend identifier of the soft-deleted payload row. /// The externalized payload token whose backing blob should be deleted. -public sealed record TombstonedPayloadDto(int PartitionId, long InstanceKey, long PayloadId, string Token); +public sealed record TombstonedPayload(int PartitionId, long InstanceKey, long PayloadId, string Token); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 822f7238..8be31fd5 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -625,17 +625,23 @@ public override async Task> GetOrchestrationHistoryAsync( } /// - public override async Task> GetTombstonedPayloadsAsync( + public override async Task> GetTombstonedPayloadsAsync( int limit, CancellationToken cancellation = default) { + if (limit <= 0 || limit >= 1000) + { + throw new ArgumentOutOfRangeException( + nameof(limit), limit, "Limit must be greater than 0 and less than 1000."); + } + P.GetTombstonedPayloadsResponse response = await this.sidecarClient.GetTombstonedPayloadsAsync( new P.GetTombstonedPayloadsRequest { Limit = limit }, cancellationToken: cancellation); - List result = new(response.Payloads.Count); + List result = new(response.Payloads.Count); foreach (P.TombstonedPayload payload in response.Payloads) { - result.Add(new TombstonedPayloadDto( + result.Add(new TombstonedPayload( payload.PartitionId, payload.InstanceKey, payload.PayloadId, payload.Token)); } @@ -644,12 +650,12 @@ public override async Task> GetTombstonedPayloadsAsyn /// public override async Task AckPurgedPayloadsAsync( - IEnumerable acks, CancellationToken cancellation = default) + IEnumerable acks, CancellationToken cancellation = default) { Check.NotNull(acks); P.AckPurgedPayloadsRequest request = new(); - foreach (PayloadPurgeAckDto ack in acks) + foreach (PayloadPurgeAck ack in acks) { request.Acks.Add(new P.PayloadPurgeAck { diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs index 637fa5ec..7ef6dbc8 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs @@ -16,13 +16,13 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class AckPurgedPayloadsActivity( DurableTaskClient client, ILogger logger) - : TaskActivity, object?> + : TaskActivity, object?> { readonly DurableTaskClient client = Check.NotNull(client); readonly ILogger logger = Check.NotNull(logger); /// - public override async Task RunAsync(TaskActivityContext context, List input) + public override async Task RunAsync(TaskActivityContext context, List input) { if (input is null || input.Count == 0) { diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 0018cb7c..f070f964 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -7,8 +7,8 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Activity that deletes a single externalized payload blob given its token. Deletion is idempotent, so -/// re-delivered tokens and concurrent workers are safe. On failure the payload is left tombstoned so a later -/// purge cycle can retry it. +/// re-delivered tokens and concurrent workers are safe. Malformed (poison) tokens are discarded so they get +/// acknowledged instead of retried forever; transient failures leave the payload tombstoned to retry. /// /// The payload store used to delete blobs. /// The logger instance. @@ -16,27 +16,34 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class DeleteExternalBlobActivity( PayloadStore store, ILogger logger) - : TaskActivity + : TaskActivity { readonly PayloadStore store = Check.NotNull(store); readonly ILogger logger = Check.NotNull(logger); /// - public override async Task RunAsync(TaskActivityContext context, string input) + public override async Task RunAsync(TaskActivityContext context, string input) { Check.NotNullOrEmpty(input, nameof(input)); try { await this.store.DeleteAsync(input, CancellationToken.None); - return true; + return BlobDeleteResult.Deleted; + } + catch (ArgumentException ex) + { + // The token is malformed or points at a different container; it can never succeed. Discard it so + // the backend clears the row instead of re-streaming the same poison token every cycle. + this.logger.BlobPurgeDeleteDiscarded(ex, input); + return BlobDeleteResult.Discarded; } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { - // Leave the payload tombstoned so the backend re-streams it on a later cycle; a single bad token - // must not fail the whole batch. + // Transient failure: leave the payload tombstoned so a later purge cycle can retry it. A single + // bad token must not fail the whole batch. this.logger.BlobPurgeDeleteFailed(ex, input); - return false; + return BlobDeleteResult.Retry; } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs index 6086efe2..2b1bad7d 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs @@ -15,16 +15,16 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class GetTombstonedPayloadsActivity( DurableTaskClient client, ILogger logger) - : TaskActivity> + : TaskActivity> { readonly DurableTaskClient client = Check.NotNull(client); readonly ILogger logger = Check.NotNull(logger); /// - public override async Task> RunAsync(TaskActivityContext context, int input) + public override async Task> RunAsync(TaskActivityContext context, int input) { - int limit = input > 0 ? input : 500; - List payloads = + int limit = input > 0 ? input : BlobPurgeConstants.DefaultBatchSize; + List payloads = await this.client.GetTombstonedPayloadsAsync(limit, CancellationToken.None); this.logger.BlobPurgeFetchedTombstones(payloads.Count); return payloads; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 9652c1e3..17d11ab0 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -51,7 +50,7 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : 500; + int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend // is reachable. @@ -79,19 +78,16 @@ async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) { try { - if (await this.IsJobActiveAsync(cancellationToken)) - { - return; - } - + // The singleton is already guaranteed by the entity's fixed key (Create no-ops when the job is + // active) and the orchestrator's fixed instance id, so just schedule the bridge once with a + // fixed instance id. Retry only if the backend is unreachable at startup. BlobPurgeJobOperationRequest request = new( - this.entityId, - nameof(BlobPurgeJob.Create), - new BlobPurgeJobCreationOptions(batchSize)); + this.entityId, nameof(BlobPurgeJob.Create), batchSize); await this.client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, + new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId), cancellationToken); this.logger.BlobPurgeJobEnsured(); @@ -115,21 +111,4 @@ await this.client.ScheduleNewOrchestrationInstanceAsync( } } } - - async Task IsJobActiveAsync(CancellationToken cancellationToken) - { - try - { - EntityMetadata? metadata = - await this.client.Entities.GetEntityAsync( - this.entityId, cancellation: cancellationToken); - return metadata is not null && metadata.State.Status == BlobPurgeJobStatus.Active; - } - catch (NotSupportedException) - { - // The entity-query API is unavailable on this client; fall back to scheduling the idempotent - // Create, which no-ops if the job is already active. - return false; - } - } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index eafee991..3697f199 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -14,6 +14,18 @@ static class BlobPurgeConstants /// public const string JobId = "__dt_blob_payload_autopurge__"; + /// + /// The default number of tombstoned payloads the auto-purge job requests from the backend per cycle, + /// used whenever a configured batch size is missing or non-positive. + /// + public const int DefaultBatchSize = 500; + + /// + /// The fixed instance ID of the client-to-entity bridge orchestration the starter schedules to ensure the + /// singleton job. A fixed ID keeps racing client processes from creating duplicate bridge orchestrations. + /// + public const string StarterInstanceId = "BlobPurgeJobStarter-" + JobId; + /// /// The prefix used for generating blob purge job orchestrator instance IDs. Format: "BlobPurgeJob-{jobId}". /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 8f08b867..c2103fa6 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -18,11 +18,11 @@ class BlobPurgeJob(ILogger logger) : TaskEntity /// client processes racing to create it do not disturb the running job. /// /// The entity context. - /// The job creation options. - public void Create(TaskEntityContext context, BlobPurgeJobCreationOptions creationOptions) + /// + /// The maximum number of tombstoned payloads to request from the backend per cycle. + /// + public void Create(TaskEntityContext context, int purgeBatchSize) { - Check.NotNull(creationOptions, nameof(creationOptions)); - if (this.State.Status == BlobPurgeJobStatus.Active) { logger.BlobPurgeJobAlreadyRunning(context.Id.Key); @@ -30,7 +30,7 @@ public void Create(TaskEntityContext context, BlobPurgeJobCreationOptions creati } this.State.Status = BlobPurgeJobStatus.Active; - this.State.PurgeBatchSize = creationOptions.PurgeBatchSize > 0 ? creationOptions.PurgeBatchSize : 500; + this.State.PurgeBatchSize = purgeBatchSize > 0 ? purgeBatchSize : BlobPurgeConstants.DefaultBatchSize; this.State.CreatedAt ??= DateTimeOffset.UtcNow; this.State.LastModifiedAt = DateTimeOffset.UtcNow; this.State.LastError = null; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 6e4ebed8..3b7ccc98 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -36,4 +36,10 @@ static partial class Logs [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); + + [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Discarding poison externalized payload token '{token}'; it can never be deleted, acknowledging it so the backend can clear the row.")] + public static partial void BlobPurgeDeleteDiscarded(this ILogger logger, Exception exception, string token); + + [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] + public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs new file mode 100644 index 00000000..d3386be5 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// The outcome of attempting to delete a single externalized payload blob during an auto-purge cycle. +/// +public enum BlobDeleteResult +{ + /// + /// The blob was deleted, or was already gone. The payload should be acknowledged so the backend can + /// hard-delete the row. + /// + Deleted, + + /// + /// The token is permanently invalid (poison) and can never be deleted. The payload should still be + /// acknowledged so the backend clears the stuck row instead of re-streaming the same token forever. + /// + Discarded, + + /// + /// A transient failure occurred. The payload is left tombstoned so a later purge cycle can retry it. + /// + Retry, +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs deleted file mode 100644 index e49e423c..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// Options used to create the singleton blob payload auto-purge job. -/// -/// -/// The maximum number of tombstoned payloads to request from the backend per cycle. -/// -public sealed record BlobPurgeJobCreationOptions(int PurgeBatchSize); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs index 917b8d1b..97c37e66 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs @@ -9,18 +9,13 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public enum BlobPurgeJobStatus { /// - /// The job is not running. This is the default status of a freshly initialized entity, so it is kept - /// as the zero value to avoid a brand-new entity accidentally appearing active. + /// The job has not been started yet. This is the default status of a freshly initialized entity, so it is + /// kept as the zero value to avoid a brand-new entity accidentally appearing active. /// - Stopped, + Pending, /// /// The job is active and draining tombstoned payloads from the backend. /// Active, - - /// - /// The job has failed. - /// - Failed, } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 2b7ba6f2..bf3db1d4 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -26,8 +26,8 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator(); string jobId = input.JobEntityId.Key; - int batchSize = input.PurgeBatchSize > 0 ? input.PurgeBatchSize : DefaultPurgeBatchSize; + int batchSize = input.PurgeBatchSize > 0 ? input.PurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; int processedCycles = input.ProcessedCycles; while (true) @@ -54,92 +54,103 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator( - input.JobEntityId, nameof(BlobPurgeJob.Get), null); - - if (state is null || state.Status != BlobPurgeJobStatus.Active) + try { - logger.BlobPurgeJobOrchestratorStopping(jobId, state?.Status.ToString() ?? "null"); - return null; - } - - List tombstones = await context.CallActivityAsync>( - nameof(GetTombstonedPayloadsActivity), - batchSize, - new TaskOptions(PurgeActivityRetryPolicy)); + // Stop cleanly if the job has been stopped or removed. + BlobPurgeJobState? state = await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.Get), null); + + if (state is null || state.Status != BlobPurgeJobStatus.Active) + { + logger.BlobPurgeJobOrchestratorStopping(jobId, state?.Status.ToString() ?? "null"); + return null; + } + + List tombstones = await context.CallActivityAsync>( + nameof(GetTombstonedPayloadsActivity), + batchSize, + new TaskOptions(PurgeActivityRetryPolicy)); - if (tombstones is null || tombstones.Count == 0) - { - // Nothing to purge right now: block on a timer (push-free idle) then check again. - await context.CreateTimer(IdleDelay, default); - continue; + if (tombstones is null || tombstones.Count == 0) + { + // Nothing to purge right now: block on a timer (push-free idle) then check again. + await context.CreateTimer(IdleDelay, default); + continue; + } + + List acks = await this.DeleteBatchAsync(context, tombstones); + + if (acks.Count > 0) + { + await context.CallActivityAsync( + nameof(AckPurgedPayloadsActivity), + acks, + new TaskOptions(PurgeActivityRetryPolicy)); + + await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)acks.Count); + } } - - List deleted = await this.DeleteBatchAsync(context, tombstones); - - if (deleted.Count > 0) + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { - await context.CallActivityAsync( - nameof(AckPurgedPayloadsActivity), - deleted, - new TaskOptions(PurgeActivityRetryPolicy)); - - await context.Entities.CallEntityAsync( - input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)deleted.Count); + // A single bad cycle (transient backend/entity/activity failure) must not kill the perpetual + // loop. Log, back off, then continue so the job self-heals and keeps draining. + logger.BlobPurgeCycleFailed(ex, jobId); + await context.CreateTimer(ErrorBackoff, default); + continue; } } } - async Task> DeleteBatchAsync( - TaskOrchestrationContext context, List tombstones) + async Task> DeleteBatchAsync( + TaskOrchestrationContext context, List tombstones) { - List deleted = new(tombstones.Count); + List acks = new(tombstones.Count); List> tasks = new(); - foreach (TombstonedPayloadDto tombstone in tombstones) + foreach (TombstonedPayload tombstone in tombstones) { tasks.Add(this.DeleteOneAsync(context, tombstone)); if (tasks.Count >= MaxParallelDeletes) { - await DrainAsync(tasks, deleted); + await DrainAsync(tasks, acks); tasks.Clear(); } } if (tasks.Count > 0) { - await DrainAsync(tasks, deleted); + await DrainAsync(tasks, acks); } - return deleted; + return acks; } - static async Task DrainAsync(List> tasks, List deleted) + static async Task DrainAsync(List> tasks, List acks) { DeleteOutcome[] outcomes = await Task.WhenAll(tasks); foreach (DeleteOutcome outcome in outcomes) { - // Only acknowledge blobs that were actually deleted; failed tokens stay tombstoned to retry. - if (outcome.Deleted) + // Acknowledge blobs that were deleted (or already gone) and poison tokens that can never succeed + // so the backend can hard-delete their rows; transient failures stay tombstoned to retry. + if (outcome.ShouldAck) { - deleted.Add(outcome.Ack); + acks.Add(outcome.Ack); } } } - async Task DeleteOneAsync(TaskOrchestrationContext context, TombstonedPayloadDto tombstone) + async Task DeleteOneAsync(TaskOrchestrationContext context, TombstonedPayload tombstone) { - bool deleted = await context.CallActivityAsync( + BlobDeleteResult result = await context.CallActivityAsync( nameof(DeleteExternalBlobActivity), - tombstone.Token, - new TaskOptions(PurgeActivityRetryPolicy)); + tombstone.Token); return new DeleteOutcome( - deleted, - new PayloadPurgeAckDto(tombstone.PartitionId, tombstone.InstanceKey, tombstone.PayloadId)); + result != BlobDeleteResult.Retry, + new PayloadPurgeAck(tombstone.PartitionId, tombstone.InstanceKey, tombstone.PayloadId)); } - readonly record struct DeleteOutcome(bool Deleted, PayloadPurgeAckDto Ack); + readonly record struct DeleteOutcome(bool ShouldAck, PayloadPurgeAck Ack); } diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index f53c9e72..582e3326 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using Azure.Core; +using Microsoft.DurableTask.AzureBlobPayloads; // Intentionally no DataAnnotations to avoid extra package requirements in minimal hosts. namespace Microsoft.DurableTask; @@ -128,5 +129,5 @@ public int ThresholdBytes /// Gets or sets the maximum number of tombstoned payloads the auto-purge job requests from the backend /// per cycle. Defaults to 500. Values less than or equal to zero are treated as the default. /// - public int PayloadPurgeBatchSize { get; set; } = 500; + public int PayloadPurgeBatchSize { get; set; } = BlobPurgeConstants.DefaultBatchSize; } diff --git a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 126afdf8..f3a727d6 100644 --- a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -19,7 +19,7 @@ public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() TestEntityOperation operation = new( nameof(BlobPurgeJob.Create), new TestEntityState(null), - new BlobPurgeJobCreationOptions(250)); + 250); // Act await this.job.RunAsync(operation); @@ -45,7 +45,7 @@ public async Task Create_WhenAlreadyActive_IsNoOp() TestEntityOperation operation = new( nameof(BlobPurgeJob.Create), new TestEntityState(existing), - new BlobPurgeJobCreationOptions(999)); + 999); // Act await this.job.RunAsync(operation); @@ -64,7 +64,7 @@ public async Task Create_WithNonPositiveBatchSize_FallsBackToDefault() TestEntityOperation operation = new( nameof(BlobPurgeJob.Create), new TestEntityState(null), - new BlobPurgeJobCreationOptions(0)); + 0); // Act await this.job.RunAsync(operation); From 149c63a9f06ba101bccc701680a7e9fe39d2edfe Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 10:44:23 -0700 Subject: [PATCH 03/47] Refine BlobPurgeJobStarter: pre-check bridge status before rescheduling Avoid needlessly re-running a Completed bridge orchestration on every host restart (fixed id + no dedupe means the backend would purge+replace a terminal instance). Check the existing bridge via GetInstanceAsync and only (re)schedule when it is absent or ended Failed/Terminated, so a failed setup still self-heals. Handle the schedule race with OrchestrationAlreadyExistsException. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 17d11ab0..33a9adec 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using DurableTask.Core.Exceptions; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Hosting; @@ -79,8 +80,24 @@ async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) try { // The singleton is already guaranteed by the entity's fixed key (Create no-ops when the job is - // active) and the orchestrator's fixed instance id, so just schedule the bridge once with a - // fixed instance id. Retry only if the backend is unreachable at startup. + // active) and the orchestrator's fixed instance id. The bridge orchestration's only job is to + // apply the entity's Create once, under a fixed instance id. Before (re)scheduling it, check the + // existing bridge: if it already Completed - or is still alive (Running/Pending/Suspended) - the + // job is set up, so do not reschedule. (Re-running a Completed bridge is wasteful: with a fixed + // id and no dedupe policy the backend would purge and replace the terminal instance on every + // host restart.) Only (re)schedule when the bridge is absent, or ended in a Failed/Terminated + // state that may never have applied Create - which lets a failed setup self-heal. + OrchestrationMetadata? existing = await this.client.GetInstanceAsync( + BlobPurgeConstants.StarterInstanceId, cancellationToken); + + bool needsSchedule = existing is null + or { RuntimeStatus: OrchestrationRuntimeStatus.Failed or OrchestrationRuntimeStatus.Terminated }; + if (!needsSchedule) + { + this.logger.BlobPurgeJobEnsured(); + return; + } + BlobPurgeJobOperationRequest request = new( this.entityId, nameof(BlobPurgeJob.Create), batchSize); @@ -93,6 +110,13 @@ await this.client.ScheduleNewOrchestrationInstanceAsync( this.logger.BlobPurgeJobEnsured(); return; } + catch (OrchestrationAlreadyExistsException) + { + // Race: another client scheduled the bridge between our status check and schedule call. That is + // fine - the singleton is already kicked off; treat it as ensured and stop. + this.logger.BlobPurgeJobEnsured(); + return; + } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { return; From 4d52005c80e35518c7cef3feba2c4f1e5de50b2d Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 11:19:09 -0700 Subject: [PATCH 04/47] Classify RequestFailedException 400 as permanent in DeleteExternalBlobActivity The Azure Storage SDK already retries transient failures internally (connection errors + HTTP 408/429/5xx with backoff), so an escaped exception means those retries were exhausted. Treating every escaped exception as Retry mis-classified permanent service rejections (e.g. Status 400 InvalidResourceName from a malformed decoded blob name) as transient, causing an infinite re-drain of a poison token. Add a RequestFailedException Status 400 -> Discarded branch (ack so the backend clears the row); keep 403/408/429/5xx/timeouts/cancellation as Retry. Document the doc-verified exception model on the activity and add focused tests for the 400 -> Discarded and non-400 -> Retry outcomes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Activities/DeleteExternalBlobActivity.cs | 36 +++++++++++- .../DeleteExternalBlobActivityTests.cs | 58 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 test/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index f070f964..6452857c 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -1,15 +1,39 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Azure; using Microsoft.Extensions.Logging; namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Activity that deletes a single externalized payload blob given its token. Deletion is idempotent, so -/// re-delivered tokens and concurrent workers are safe. Malformed (poison) tokens are discarded so they get -/// acknowledged instead of retried forever; transient failures leave the payload tombstoned to retry. +/// re-delivered tokens and concurrent workers are safe. /// +/// +/// Outcome classification, verified against the Azure.Storage.Blobs / Azure.Core exception model (not +/// assumed): +/// +/// +/// The Azure SDK already retries transient failures internally (connection errors plus HTTP +/// 408/429/500/502/503/504, with exponential backoff), so any exception that escapes +/// means those built-in retries were already exhausted. +/// +/// +/// Permanent failures are discarded (acked so the backend clears the row): an +/// from the store's own token decode / container-mismatch check (thrown client-side before any network call), +/// and a with 400 (for +/// example InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules). Retrying +/// either can never succeed. +/// +/// +/// Everything else is treated as transient and leaves the payload tombstoned to retry on a later cycle: +/// throttling / 5xx that outlived the SDK's retries, 403 authorization failures (which need an operator +/// credential fix rather than dropping data), and timeouts / cancellation. A blob is never dropped on an +/// uncertain error, and a single bad token never fails the whole batch. +/// +/// +/// /// The payload store used to delete blobs. /// The logger instance. [DurableTask] @@ -38,6 +62,14 @@ public override async Task RunAsync(TaskActivityContext contex this.logger.BlobPurgeDeleteDiscarded(ex, input); return BlobDeleteResult.Discarded; } + catch (RequestFailedException ex) when (ex.Status == 400) + { + // Service rejected the request as permanently invalid (e.g. InvalidUri / InvalidResourceName - the + // decoded blob name violates Azure naming rules). Retrying can never succeed, so discard it like a + // poison token: ack so the backend clears the row instead of re-streaming it forever. + this.logger.BlobPurgeDeleteDiscarded(ex, input); + return BlobDeleteResult.Discarded; + } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // Transient failure: leave the payload tombstoned so a later purge cycle can retry it. A single diff --git a/test/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs new file mode 100644 index 00000000..6ca963cb --- /dev/null +++ b/test/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure; +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class DeleteExternalBlobActivityTests +{ + [Fact] + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_DiscardsPoisonToken() + { + // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. + StubPayloadStore store = new(new RequestFailedException(400, "InvalidResourceName")); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:bad name"); + + // Assert - discarded so the backend acks and clears the row instead of re-streaming forever. + result.Should().Be(BlobDeleteResult.Discarded); + } + + [Fact] + public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_LeavesTombstonedForRetry() + { + // Arrange - a Status 503 that escaped the SDK's internal retries is treated as transient. + StubPayloadStore store = new(new RequestFailedException(503, "ServerBusy")); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + + // Assert - left tombstoned so a later purge cycle can retry; a blob is never dropped on doubt. + result.Should().Be(BlobDeleteResult.Retry); + } + + sealed class StubPayloadStore : PayloadStore + { + readonly Exception? deleteError; + + public StubPayloadStore(Exception? deleteError) => this.deleteError = deleteError; + + public override Task DeleteAsync(string token, CancellationToken cancellationToken) => + this.deleteError is null ? Task.CompletedTask : throw this.deleteError; + + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => true; + } +} From 780d7430ae286f533c87c9b01f01f98cdb482a26 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 13:25:12 -0700 Subject: [PATCH 05/47] Reuse shared PayloadStore and register purge starter conditionally on AutoPurge Address review #B (store duplication) and #C (conditional-DI of the starter): - TryAddSingleton the PayloadStore so we reuse an already-registered shared store instead of creating a second, redundant one. - Register the auto-purge starter only when the configure overload opted into AutoPurge (peek the flag via a probe options instance); UseExternalizedPayloadsCore no longer registers it unconditionally, so the no-arg overload never starts it. - The starter no longer self-checks the flag in StartAsync (registration already gates it); remove the now-unused BlobPurgeDisabled log and refresh its xmldoc. - Add DI-registration tests proving the starter is registered iff AutoPurge is on. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 15 ++--- .../AzureBlobPayloads/AutoPurge/Logs.cs | 3 - ...ientBuilderExtensions.AzureBlobPayloads.cs | 22 +++++-- .../AzureBlobPayloads.Tests.csproj | 1 + .../UseExternalizedPayloadsTests.cs | 62 +++++++++++++++++++ 5 files changed, 86 insertions(+), 17 deletions(-) create mode 100644 test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 33a9adec..23176ef8 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -11,10 +11,11 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Client-side hosted service that ensures the singleton blob payload auto-purge job exists when auto-purge -/// is enabled via . It never blocks host startup: it runs -/// on a background task and retries until the backend is reachable. The job is a whole-scheduler singleton, -/// so racing client processes simply no-op. +/// Client-side hosted service that ensures the singleton blob payload auto-purge job exists. It is registered +/// only when auto-purge is enabled at registration time (see the UseExternalizedPayloads configure overload), +/// so it does not re-check the flag here. It never blocks host startup: it runs on a background task and +/// retries until the backend is reachable. The job is a whole-scheduler singleton, so racing client processes +/// simply no-op. /// sealed class BlobPurgeJobStarter : IHostedService { @@ -45,12 +46,6 @@ public BlobPurgeJobStarter( public Task StartAsync(CancellationToken cancellationToken) { LargePayloadStorageOptions opts = this.options.Get(this.builderName); - if (!opts.AutoPurge) - { - this.logger.BlobPurgeDisabled(); - return Task.CompletedTask; - } - int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 3b7ccc98..b217f53e 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -28,9 +28,6 @@ static partial class Logs [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge acknowledged {count} purged payload(s) to the backend.")] public static partial void BlobPurgeAckedPayloads(this ILogger logger, int count); - [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge is disabled; the singleton purge job will not be started.")] - public static partial void BlobPurgeDisabled(this ILogger logger); - [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] public static partial void BlobPurgeJobEnsured(this ILogger logger); diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 2557c6da..78e4d47f 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -8,6 +8,7 @@ using Microsoft.DurableTask.Converters; using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -33,13 +34,28 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( Check.NotNull(configure); builder.Services.Configure(builder.Name, configure); - builder.Services.AddSingleton(sp => + + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the worker builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => { LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); return new BlobPayloadStore(opts); }); - return UseExternalizedPayloadsCore(builder); + UseExternalizedPayloadsCore(builder); + + // Conditional DI: register the auto-purge starter only when the caller opted into auto-purge. Peek the + // flag now by running the configure delegate against a probe (options configurators are pure setters). + LargePayloadStorageOptions probe = new(); + configure(probe); + if (probe.AutoPurge) + { + RegisterBlobPurgeJobStarter(builder); + } + + return builder; } /// @@ -82,8 +98,6 @@ static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientB } }); - RegisterBlobPurgeJobStarter(builder); - return builder; } diff --git a/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj b/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj index 77078cbb..b650cb59 100644 --- a/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj +++ b/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj @@ -14,6 +14,7 @@ + diff --git a/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs new file mode 100644 index 00000000..b3343078 --- /dev/null +++ b/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.DependencyInjection; + +public class UseExternalizedPayloadsTests +{ + [Fact] + public void UseExternalizedPayloads_WithAutoPurgeEnabled_RegistersHostedPurgeStarter() + { + // Arrange + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act + builder.Object.UseExternalizedPayloads(options => options.AutoPurge = true); + + // Assert - the purge-job starter is the only IHostedService this path registers. + services.Should().ContainSingle(d => d.ServiceType == typeof(IHostedService)); + } + + [Fact] + public void UseExternalizedPayloads_WithAutoPurgeDisabled_DoesNotRegisterHostedPurgeStarter() + { + // Arrange + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act - auto-purge left at its default (false). + builder.Object.UseExternalizedPayloads(options => { }); + + // Assert + services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + } + + [Fact] + public void UseExternalizedPayloads_NoArgOverload_DoesNotRegisterHostedPurgeStarter() + { + // Arrange + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act - the shared-store overload never enables auto-purge. + builder.Object.UseExternalizedPayloads(); + + // Assert + services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + } +} From 3a2215cd9796268ef56064f4603f1a3964fdc17f Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 14:07:32 -0700 Subject: [PATCH 06/47] Register fallback PayloadStore in shared Core for both client and worker The store's consumer is the interceptor wired up in UseExternalizedPayloadsCore, so Core is the single fallback registration site. Move the TryAddSingleton out of each configure overload and into Core, symmetrically for the client and worker extensions. TryAdd keeps reusing a shared store (AddExternalizedPayloadStore or the sibling builder) and never creates a redundant one. The worker overload switches from AddSingleton to the shared TryAddSingleton in Core as part of the move. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...lientBuilderExtensions.AzureBlobPayloads.cs | 18 +++++++++--------- ...orkerBuilderExtensions.AzureBlobPayloads.cs | 15 ++++++++++----- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 78e4d47f..0c485208 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -35,15 +35,6 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( builder.Services.Configure(builder.Name, configure); - // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or - // the worker builder in the same process); only register our own as a fallback so we never create a - // second, redundant PayloadStore. - builder.Services.TryAddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); - UseExternalizedPayloadsCore(builder); // Conditional DI: register the auto-purge starter only when the caller opted into auto-purge. Peek the @@ -73,6 +64,15 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientBuilder builder) { + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the worker builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + // Wrap the gRPC CallInvoker with our interceptor when using the gRPC client builder.Services .AddOptions(builder.Name) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index 42ac0f0a..c63db137 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -6,6 +6,7 @@ using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; using P = Microsoft.DurableTask.Protobuf; @@ -30,11 +31,6 @@ public static IDurableTaskWorkerBuilder UseExternalizedPayloads( Check.NotNull(configure); builder.Services.Configure(builder.Name, configure); - builder.Services.AddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); return UseExternalizedPayloadsCore(builder); } @@ -54,6 +50,15 @@ public static IDurableTaskWorkerBuilder UseExternalizedPayloads( static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerBuilder builder) { + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the client builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + // Wrap the gRPC CallInvoker with our interceptor when using the gRPC worker builder.Services .AddOptions(builder.Name) From 47651dc0e93d17a67dc145f14d028df019782590 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 14:39:58 -0700 Subject: [PATCH 07/47] Stop self-registering PayloadStore on the client; consume the shared store Restore the original design: the client never registers a PayloadStore, it only consumes a shared/external one (via AddExternalizedPayloadStore or an in-process worker). Only the worker self-registers a fallback store. Remove the client Core's TryAddSingleton block (and its now-unused Microsoft.Extensions.DependencyInjection.Extensions using); the client Core PostConfigure still resolves PayloadStore from the shared/worker registration. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...bleTaskClientBuilderExtensions.AzureBlobPayloads.cs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 0c485208..f811d6da 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -8,7 +8,6 @@ using Microsoft.DurableTask.Converters; using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -64,15 +63,6 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientBuilder builder) { - // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or - // the worker builder in the same process); only register our own as a fallback so we never create a - // second, redundant PayloadStore. - builder.Services.TryAddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); - // Wrap the gRPC CallInvoker with our interceptor when using the gRPC client builder.Services .AddOptions(builder.Name) From 7397fa64e71b88150087b551580944dce0e8cc2f Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 15 Jul 2026 10:02:11 -0700 Subject: [PATCH 08/47] Validate PayloadPurgeBatchSize once at specification (fail fast on out-of-range); drop redundant downstream coercions Validate the batch size at its single point of specification - LargePayloadStorageOptions.PayloadPurgeBatchSize - and fail fast when out of range, mirroring the existing ThresholdBytes setter. Valid range is 1..999 inclusive, matching the gRPC GetTombstonedPayloadsAsync contract (which rejects limits >= 1000); add BlobPurgeConstants.MaxBatchSize = 999 to express the upper bound. Now that the value is guaranteed valid at specification, remove the redundant '> 0 ? x : DefaultBatchSize' coercions from every downstream consumer (starter, entity, orchestrator, activity); they simply use the value directly. Tests: add setter validation tests (throws for 0, -1, 1000, 1001; accepts 1, 500, 999) and repurpose the entity's non-positive test to assert the entity now stores the batch size verbatim (no coercion). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../GetTombstonedPayloadsActivity.cs | 2 +- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 2 +- .../AutoPurge/Constants/BlobPurgeConstants.cs | 8 ++++- .../AutoPurge/Entity/BlobPurgeJob.cs | 2 +- .../BlobPurgeJobOrchestrator.cs | 2 +- .../Options/LargePayloadStorageOptions.cs | 20 +++++++++-- .../AutoPurge/BlobPurgeJobTests.cs | 8 +++-- .../LargePayloadStorageOptionsTests.cs | 33 +++++++++++++++++++ 8 files changed, 67 insertions(+), 10 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs index 2b1bad7d..838b293a 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs @@ -23,7 +23,7 @@ public class GetTombstonedPayloadsActivity( /// public override async Task> RunAsync(TaskActivityContext context, int input) { - int limit = input > 0 ? input : BlobPurgeConstants.DefaultBatchSize; + int limit = input; List payloads = await this.client.GetTombstonedPayloadsAsync(limit, CancellationToken.None); this.logger.BlobPurgeFetchedTombstones(payloads.Count); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 23176ef8..bc3e18a5 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -46,7 +46,7 @@ public BlobPurgeJobStarter( public Task StartAsync(CancellationToken cancellationToken) { LargePayloadStorageOptions opts = this.options.Get(this.builderName); - int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; + int batchSize = opts.PayloadPurgeBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend // is reachable. diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index 3697f199..48174f52 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -16,10 +16,16 @@ static class BlobPurgeConstants /// /// The default number of tombstoned payloads the auto-purge job requests from the backend per cycle, - /// used whenever a configured batch size is missing or non-positive. + /// used whenever a batch size is not explicitly configured. /// public const int DefaultBatchSize = 500; + /// + /// The maximum batch size the auto-purge job may request per cycle. Mirrors the gRPC + /// GetTombstonedPayloadsAsync contract, which rejects limits >= 1000. + /// + public const int MaxBatchSize = 999; + /// /// The fixed instance ID of the client-to-entity bridge orchestration the starter schedules to ensure the /// singleton job. A fixed ID keeps racing client processes from creating duplicate bridge orchestrations. diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index c2103fa6..89391890 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -30,7 +30,7 @@ public void Create(TaskEntityContext context, int purgeBatchSize) } this.State.Status = BlobPurgeJobStatus.Active; - this.State.PurgeBatchSize = purgeBatchSize > 0 ? purgeBatchSize : BlobPurgeConstants.DefaultBatchSize; + this.State.PurgeBatchSize = purgeBatchSize; this.State.CreatedAt ??= DateTimeOffset.UtcNow; this.State.LastModifiedAt = DateTimeOffset.UtcNow; this.State.LastError = null; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index bf3db1d4..8ba4f707 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -42,7 +42,7 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator(); string jobId = input.JobEntityId.Key; - int batchSize = input.PurgeBatchSize > 0 ? input.PurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; + int batchSize = input.PurgeBatchSize; int processedCycles = input.ProcessedCycles; while (true) diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 582e3326..cbd63835 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -28,6 +28,7 @@ namespace Microsoft.DurableTask; public sealed class LargePayloadStorageOptions { int thresholdBytes = 256 * 1024; + int payloadPurgeBatchSize = BlobPurgeConstants.DefaultBatchSize; /// /// Initializes a new instance of the class. @@ -127,7 +128,22 @@ public int ThresholdBytes /// /// Gets or sets the maximum number of tombstoned payloads the auto-purge job requests from the backend - /// per cycle. Defaults to 500. Values less than or equal to zero are treated as the default. + /// per cycle. Must be between 1 and 999 (inclusive); values outside this range throw + /// . Defaults to 500. /// - public int PayloadPurgeBatchSize { get; set; } = BlobPurgeConstants.DefaultBatchSize; + public int PayloadPurgeBatchSize + { + get => this.payloadPurgeBatchSize; + set + { + if (value < 1 || value > BlobPurgeConstants.MaxBatchSize) + { + throw new ArgumentOutOfRangeException( + nameof(this.PayloadPurgeBatchSize), value, + $"PayloadPurgeBatchSize must be between 1 and {BlobPurgeConstants.MaxBatchSize} (inclusive)."); + } + + this.payloadPurgeBatchSize = value; + } + } } diff --git a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index f3a727d6..2117299c 100644 --- a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -58,9 +58,11 @@ public async Task Create_WhenAlreadyActive_IsNoOp() } [Fact] - public async Task Create_WithNonPositiveBatchSize_FallsBackToDefault() + public async Task Create_StoresBatchSizeVerbatim_WithoutCoercion() { - // Arrange + // Arrange - the batch size is validated once at specification (LargePayloadStorageOptions), so the + // entity trusts its input and performs no coercion of its own. A zero here is stored as-is, proving + // the previous non-positive-to-default fallback was removed. TestEntityOperation operation = new( nameof(BlobPurgeJob.Create), new TestEntityState(null), @@ -72,7 +74,7 @@ public async Task Create_WithNonPositiveBatchSize_FallsBackToDefault() // Assert BlobPurgeJobState state = Assert.IsType( operation.State.GetState(typeof(BlobPurgeJobState))); - state.PurgeBatchSize.Should().Be(500); + state.PurgeBatchSize.Should().Be(0); } [Fact] diff --git a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs index 317b1e1e..6a4a95ed 100644 --- a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs +++ b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs @@ -18,4 +18,37 @@ public void Defaults_AutoPurgeDisabled_AndBatchSize500() options.AutoPurge.Should().BeFalse(); options.PayloadPurgeBatchSize.Should().Be(500); } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1000)] + [InlineData(1001)] + public void PayloadPurgeBatchSize_OutOfRange_Throws(int value) + { + // Arrange + LargePayloadStorageOptions options = new(); + + // Act + Action act = () => options.PayloadPurgeBatchSize = value; + + // Assert + act.Should().Throw(); + } + + [Theory] + [InlineData(1)] + [InlineData(500)] + [InlineData(999)] + public void PayloadPurgeBatchSize_InRange_IsAccepted(int value) + { + // Arrange + LargePayloadStorageOptions options = new(); + + // Act + options.PayloadPurgeBatchSize = value; + + // Assert + options.PayloadPurgeBatchSize.Should().Be(value); + } } From 4afeb8a93aa05cb46a3098c4f0d54656c583c938 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 15 Jul 2026 10:03:51 -0700 Subject: [PATCH 09/47] Translate gRPC Cancelled to OperationCanceledException in GetTombstonedPayloadsAsync/AckPurgedPayloadsAsync Both new methods previously let RpcException(StatusCode.Cancelled) escape, unlike every other method in this client. Wrap each method's gRPC call in a try/catch that translates Cancelled to OperationCanceledException, mirroring the existing pattern used by GetOrchestrationHistoryAsync and the other client methods. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Client/Grpc/GrpcDurableTaskClient.cs | 25 ++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 8be31fd5..ca9b51ae 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -634,9 +634,18 @@ public override async Task> GetTombstonedPayloadsAsync( nameof(limit), limit, "Limit must be greater than 0 and less than 1000."); } - P.GetTombstonedPayloadsResponse response = await this.sidecarClient.GetTombstonedPayloadsAsync( - new P.GetTombstonedPayloadsRequest { Limit = limit }, - cancellationToken: cancellation); + P.GetTombstonedPayloadsResponse response; + try + { + response = await this.sidecarClient.GetTombstonedPayloadsAsync( + new P.GetTombstonedPayloadsRequest { Limit = limit }, + cancellationToken: cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.GetTombstonedPayloadsAsync)} operation was canceled.", e, cancellation); + } List result = new(response.Payloads.Count); foreach (P.TombstonedPayload payload in response.Payloads) @@ -670,7 +679,15 @@ public override async Task AckPurgedPayloadsAsync( return; } - await this.sidecarClient.AckPurgedPayloadsAsync(request, cancellationToken: cancellation); + try + { + await this.sidecarClient.AckPurgedPayloadsAsync(request, cancellationToken: cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.AckPurgedPayloadsAsync)} operation was canceled.", e, cancellation); + } } static AsyncDisposable GetCallInvoker(GrpcDurableTaskClientOptions options, ILogger logger, out CallInvoker callInvoker) From e74f6330ab6de3fe6accdfab1892a7e6752e5a9a Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 15 Jul 2026 11:05:17 -0700 Subject: [PATCH 10/47] Raise auto-purge MaxBatchSize to 1000 (inclusive); relax gRPC GetTombstonedPayloads limit check to allow 1000 1000 is a valid, accepted batch size end-to-end: the proto documents no numeric range and the backend only clamps via min(limit, configuredCap) without rejecting >= 1000, so the whole range is contained in this SDK. Raise BlobPurgeConstants.MaxBatchSize from 999 to 1000 (the options setter already validates against it) and relax the gRPC GetTombstonedPayloadsAsync guard from 'limit >= 1000' to 'limit > 1000'. Update the two PayloadPurgeBatchSize theories so 1000 is asserted accepted and 1001 still throws. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Client/Grpc/GrpcDurableTaskClient.cs | 4 ++-- .../AutoPurge/Constants/BlobPurgeConstants.cs | 4 ++-- .../AzureBlobPayloads/Options/LargePayloadStorageOptions.cs | 2 +- .../Options/LargePayloadStorageOptionsTests.cs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index ca9b51ae..6e79b6b2 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -628,10 +628,10 @@ public override async Task> GetOrchestrationHistoryAsync( public override async Task> GetTombstonedPayloadsAsync( int limit, CancellationToken cancellation = default) { - if (limit <= 0 || limit >= 1000) + if (limit <= 0 || limit > 1000) { throw new ArgumentOutOfRangeException( - nameof(limit), limit, "Limit must be greater than 0 and less than 1000."); + nameof(limit), limit, "Limit must be greater than 0 and less than or equal to 1000."); } P.GetTombstonedPayloadsResponse response; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index 48174f52..ca4cce32 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -22,9 +22,9 @@ static class BlobPurgeConstants /// /// The maximum batch size the auto-purge job may request per cycle. Mirrors the gRPC - /// GetTombstonedPayloadsAsync contract, which rejects limits >= 1000. + /// GetTombstonedPayloadsAsync contract, which rejects limits greater than 1000. /// - public const int MaxBatchSize = 999; + public const int MaxBatchSize = 1000; /// /// The fixed instance ID of the client-to-entity bridge orchestration the starter schedules to ensure the diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index cbd63835..0edaee90 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -128,7 +128,7 @@ public int ThresholdBytes /// /// Gets or sets the maximum number of tombstoned payloads the auto-purge job requests from the backend - /// per cycle. Must be between 1 and 999 (inclusive); values outside this range throw + /// per cycle. Must be between 1 and 1000 (inclusive); values outside this range throw /// . Defaults to 500. /// public int PayloadPurgeBatchSize diff --git a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs index 6a4a95ed..d608a936 100644 --- a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs +++ b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs @@ -22,7 +22,6 @@ public void Defaults_AutoPurgeDisabled_AndBatchSize500() [Theory] [InlineData(0)] [InlineData(-1)] - [InlineData(1000)] [InlineData(1001)] public void PayloadPurgeBatchSize_OutOfRange_Throws(int value) { @@ -40,6 +39,7 @@ public void PayloadPurgeBatchSize_OutOfRange_Throws(int value) [InlineData(1)] [InlineData(500)] [InlineData(999)] + [InlineData(1000)] public void PayloadPurgeBatchSize_InRange_IsAccepted(int value) { // Arrange From 50ae944676378bd3da37591fb16073aadce33ff0 Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 30 Jul 2026 10:54:51 -0700 Subject: [PATCH 11/47] Register PayloadStore in the client builder extension (symmetry with worker; fixes client-only DI failure) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- ...ientBuilderExtensions.AzureBlobPayloads.cs | 10 +++++++++ .../UseExternalizedPayloadsTests.cs | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index f811d6da..0c485208 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -8,6 +8,7 @@ using Microsoft.DurableTask.Converters; using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -63,6 +64,15 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientBuilder builder) { + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the worker builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + // Wrap the gRPC CallInvoker with our interceptor when using the gRPC client builder.Services .AddOptions(builder.Name) diff --git a/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index b3343078..2ee439ef 100644 --- a/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -59,4 +59,25 @@ public void UseExternalizedPayloads_NoArgOverload_DoesNotRegisterHostedPurgeStar // Assert services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); } + + [Fact] + public void UseExternalizedPayloads_ClientOnly_RegistersResolvablePayloadStore() + { + // Arrange - a client-only host with no worker and no explicit AddExternalizedPayloadStore. This is the + // exact shape that previously failed: the core method declared a PostConfigure dependency on + // PayloadStore without ever registering it, so options resolution threw at runtime. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act - UseDevelopmentStorage=true is a valid connection string that BlobServiceClient accepts with no + // network I/O, so the store constructs offline. Build the provider and actually resolve PayloadStore. + builder.Object.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - the store resolves without throwing and is the blob-backed implementation. + PayloadStore store = provider.GetRequiredService(); + store.Should().BeOfType(); + } } From a5ed2989f8e9c7d17647bd37e9f6bdad5b888e64 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 12:59:34 -0700 Subject: [PATCH 12/47] Resolve v2 tokens in DeleteAsync and discard payloads in unreachable accounts The blob delete activity now discards (acks) a payload whose v2 token points at a storage account the configured credential cannot reach: BlobPayloadStore throws PayloadStorageException for that cross-account case, and since the backend batch is a cursor-less SELECT TOP (N), retrying such a permanently unreachable row would re-stream it every cycle and block the pipeline head-of-line. It is discarded and logged at error level (EventId 821) instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../Activities/DeleteExternalBlobActivity.cs | 24 +++++++++++++++---- .../AzureBlobPayloads/AutoPurge/Logs.cs | 3 +++ .../DeleteExternalBlobActivityTests.cs | 16 +++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 6452857c..76490237 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -20,11 +20,16 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// means those built-in retries were already exhausted. /// /// -/// Permanent failures are discarded (acked so the backend clears the row): an -/// from the store's own token decode / container-mismatch check (thrown client-side before any network call), -/// and a with 400 (for -/// example InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules). Retrying -/// either can never succeed. +/// Permanent failures are discarded (acked so the backend clears the row) because retrying can never succeed: +/// an from the store's token decode - a genuinely malformed token, or a legacy +/// v1 token whose container does not match the configured store (self-describing v2 tokens are not conflated +/// into this: a different account/container is either handled or surfaced as the store exception below); a +/// with 400 (for example +/// InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules); and a +/// when a v2 token points at a storage account the configured credential +/// cannot reach (connection-string / account-key auth is account-specific). The backend batch is cursor-less, +/// so an undroppable row would otherwise re-stream every cycle and block the pipeline head-of-line; the +/// account-unreachable case is logged at error level so an operator can reconcile it. /// /// /// Everything else is treated as transient and leaves the payload tombstoned to retry on a later cycle: @@ -70,6 +75,15 @@ public override async Task RunAsync(TaskActivityContext contex this.logger.BlobPurgeDeleteDiscarded(ex, input); return BlobDeleteResult.Discarded; } + catch (PayloadStorageException ex) + { + // The payload lives in a storage account this worker's credential cannot reach (connection-string / + // account-key auth is account-specific). Retrying can never succeed and the batch is cursor-less, so a + // permanently unreachable row would re-stream every cycle and block later rows. Discard it and log + // loudly so an operator can reconfigure identity auth and delete the blob out of band. + this.logger.BlobPurgeDeleteOrphanedUnreachableAccount(ex, input); + return BlobDeleteResult.Discarded; + } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // Transient failure: leave the payload tombstoned so a later purge cycle can retry it. A single diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index b217f53e..7ef2e16b 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -39,4 +39,7 @@ static partial class Logs [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); + + [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' was orphaned: its blob lives in a storage account the configured credential cannot reach (cross-account deletes require identity/AAD auth; connection-string / account-key auth is account-specific). Discarding it so the purge pipeline is not blocked; reconfigure identity auth with access to that account and delete the blob out of band.")] + public static partial void BlobPurgeDeleteOrphanedUnreachableAccount(this ILogger logger, Exception exception, string token); } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 6ca963cb..5e3004cb 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -38,6 +38,22 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_LeavesTombstonedF result.Should().Be(BlobDeleteResult.Retry); } + [Fact] + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_DiscardsToUnblockPipeline() + { + // Arrange - the payload lives in a storage account the configured credential cannot reach. Retrying can + // never succeed and the backend batch is cursor-less, so a permanently unreachable row would re-stream + // every cycle and block later rows; it must be discarded (acked), not retried. + StubPayloadStore store = new(new PayloadStorageException("cross-account delete requires identity auth")); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); + + // Assert - discarded so the pipeline head-of-line is not blocked by an undeletable payload. + result.Should().Be(BlobDeleteResult.Discarded); + } + sealed class StubPayloadStore : PayloadStore { readonly Exception? deleteError; From fff06b02f81e593804003aed4218659c4ad75876 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 13:08:05 -0700 Subject: [PATCH 13/47] Align unreachable-account log, exception wording and v2 delete test names with review spec Rename the cross-account orphan log to BlobPurgeDeleteUnreachable (EventId 821, Error) with the reviewer-specified message, mirror that exact wording in the DeleteExternalBlobActivity PayloadStorageException catch, reword the BlobPayloadStore cross-account message ("cannot delete in another account"), and rename the three v2 DeleteAsync tests to the reviewer-specified names. No behavior change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../Activities/DeleteExternalBlobActivity.cs | 11 ++++++----- src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs | 4 ++-- .../PayloadStore/BlobPayloadStore.cs | 2 +- .../PayloadStore/BlobPayloadStoreDeleteTests.cs | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 76490237..2935cf68 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -77,11 +77,12 @@ public override async Task RunAsync(TaskActivityContext contex } catch (PayloadStorageException ex) { - // The payload lives in a storage account this worker's credential cannot reach (connection-string / - // account-key auth is account-specific). Retrying can never succeed and the batch is cursor-less, so a - // permanently unreachable row would re-stream every cycle and block later rows. Discard it and log - // loudly so an operator can reconfigure identity auth and delete the blob out of band. - this.logger.BlobPurgeDeleteOrphanedUnreachableAccount(ex, input); + // The token is well-formed but points at a storage account this worker's credential cannot reach + // (cross-account without AAD). Retrying can never succeed from this process, and because the backend + // streams tombstones with an uncursored TOP(N) query, leaving it un-acked would re-serve the same + // token every cycle and permanently block the purge pipeline. Discard it so the row is cleared, and + // log at Error so an operator can reclaim the orphaned blob out-of-band. + this.logger.BlobPurgeDeleteUnreachable(ex, input); return BlobDeleteResult.Discarded; } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 7ef2e16b..5238496d 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -40,6 +40,6 @@ static partial class Logs [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); - [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' was orphaned: its blob lives in a storage account the configured credential cannot reach (cross-account deletes require identity/AAD auth; connection-string / account-key auth is account-specific). Discarding it so the purge pipeline is not blocked; reconfigure identity auth with access to that account and delete the blob out of band.")] - public static partial void BlobPurgeDeleteOrphanedUnreachableAccount(this ILogger logger, Exception exception, string token); + [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' points at a storage account the configured credential cannot reach; the blob cannot be deleted by this worker and will be orphaned. Acknowledging it so the backend can clear the row - reclaim the blob manually or reconfigure the payload store with identity (AAD) authentication that can access both accounts.")] + public static partial void BlobPurgeDeleteUnreachable(this ILogger logger, Exception exception, string token); } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index 305c2185..7a9cbacb 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -231,7 +231,7 @@ public override async Task DeleteAsync(string token, CancellationToken cancellat $"The externalized payload lives in a different storage account ('{decoded.ContainerUri}') than the " + $"currently-configured payload store ('{this.containerClient.Uri}'). Cross-account payload deletes " + "require identity (AAD) authentication with access to both accounts; connection-string / " + - "account-key credentials are account-specific and cannot delete from another account."); + "account-key credentials are account-specific and cannot delete in another account."); } // Idempotent by design: DeleteIfExistsAsync returns false (rather than throwing) when the blob is diff --git a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs index bdf238e6..7ccab24a 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs @@ -105,7 +105,7 @@ public async Task DeleteAsync_InvalidToken_ThrowsArgumentException(string token) } [Fact] - public async Task DeleteAsync_V2TokenSameAccountAndContainer_DeletesViaConfiguredContainerClient() + public async Task DeleteAsync_V2TokenSameContainer_DeletesViaConfiguredClient() { // Arrange - a self-describing v2 token whose account+container match the configured store. The store // recognizes it via IsConfiguredContainer and deletes through the existing container client (which works @@ -125,7 +125,7 @@ public async Task DeleteAsync_V2TokenSameAccountAndContainer_DeletesViaConfigure } [Fact] - public async Task DeleteAsync_V2TokenDifferentAccountWithCredential_UsesCrossAccountClientNotConfiguredContainer() + public async Task DeleteAsync_V2TokenDifferentAccountWithCredential_DoesNotUseConfiguredContainer() { // Arrange - a v2 token pointing at a DIFFERENT account than the configured store, with identity auth // available. The store must build a BlobClient bound to the token's own account using the credential and @@ -151,7 +151,7 @@ public async Task DeleteAsync_V2TokenDifferentAccountWithCredential_UsesCrossAcc } [Fact] - public async Task DeleteAsync_V2TokenDifferentAccountWithoutCredential_ThrowsPayloadStorageException() + public async Task DeleteAsync_V2TokenDifferentAccountWithoutCredential_ThrowsPayloadStorageExceptionAndDoesNotDelete() { // Arrange - the configured store uses a connection string (account-key auth, no TokenCredential) and the // token points at a different account. Account keys are account-specific, so the delete cannot cross From 6e4f5d08568df270bf7c89ce00fd7172b9df4027 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 14:16:49 -0700 Subject: [PATCH 14/47] Gate blob auto-purge on v2 tokens and deleting stores Decision 1: the auto-purge job declines to act on legacy v1 tokens. DeleteExternalBlobActivity.RunAsync discards v1 tokens up front - logged at error as the operator recovery pointer - without calling the store, because a v1 token identifies no storage account and a delete cannot be verified. BlobPayloadStore.TokenPrefixV1 is made internal so the activity can reference it; BlobPayloadStore.DeleteAsync is unchanged and still honors v1 for direct callers. Decision 2: do not start the purge job when the registered store cannot delete. BlobPurgeJobStarter now takes the resolved PayloadStore and no-ops at startup (logged at error, does not throw) when it is not a BlobPayloadStore. Defense in depth: the activity catches NotSupportedException and returns Retry to keep the payload tombstoned rather than acking a row whose blob was never deleted. Adds Logs 822/823/824, XML doc updates, a per-task-hub singleton phrasing fix, and focused tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../Activities/DeleteExternalBlobActivity.cs | 42 ++++++++-- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 16 +++- .../AutoPurge/Entity/BlobPurgeJob.cs | 2 +- .../AzureBlobPayloads/AutoPurge/Logs.cs | 9 +++ ...ientBuilderExtensions.AzureBlobPayloads.cs | 1 + .../Options/LargePayloadStorageOptions.cs | 6 ++ .../PayloadStore/BlobPayloadStore.cs | 2 +- .../AutoPurge/BlobPurgeJobStarterTests.cs | 80 +++++++++++++++++++ .../DeleteExternalBlobActivityTests.cs | 52 +++++++++++- 9 files changed, 200 insertions(+), 10 deletions(-) create mode 100644 test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 2935cf68..12438d47 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -20,10 +20,16 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// means those built-in retries were already exhausted. /// /// +/// Legacy blob:v1: tokens are discarded without a delete attempt: a v1 token carries only a container +/// name, not the storage account, so a delete against the currently-configured account cannot be verified - if +/// the store has been repointed the delete would report success while the real blob survives elsewhere. The +/// backend ack protocol has no "skip" status (an un-acked row is re-served every cycle), so the token is acked +/// to keep the pipeline moving and logged at error level as the operator's recovery pointer. +/// +/// /// Permanent failures are discarded (acked so the backend clears the row) because retrying can never succeed: -/// an from the store's token decode - a genuinely malformed token, or a legacy -/// v1 token whose container does not match the configured store (self-describing v2 tokens are not conflated -/// into this: a different account/container is either handled or surfaced as the store exception below); a +/// an from the store's token decode - a genuinely malformed or unrecognized +/// token (v1 tokens are gated out above and never reach the store); a /// with 400 (for example /// InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules); and a /// when a v2 token points at a storage account the configured credential @@ -34,8 +40,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Everything else is treated as transient and leaves the payload tombstoned to retry on a later cycle: /// throttling / 5xx that outlived the SDK's retries, 403 authorization failures (which need an operator -/// credential fix rather than dropping data), and timeouts / cancellation. A blob is never dropped on an -/// uncertain error, and a single bad token never fails the whole batch. +/// credential fix rather than dropping data), timeouts / cancellation, and a +/// from a misconfigured store that cannot delete at all (retried, not dropped, so the work is recoverable once +/// a deleting store is registered). A blob is never dropped on an uncertain error, and a single bad token never +/// fails the whole batch. /// /// /// @@ -55,6 +63,20 @@ public override async Task RunAsync(TaskActivityContext contex { Check.NotNullOrEmpty(input, nameof(input)); + if (input.StartsWith(BlobPayloadStore.TokenPrefixV1, StringComparison.Ordinal)) + { + // Auto-purge deliberately does not act on legacy v1 tokens. A v1 token carries only a container + // *name* - not the storage account - so a delete against the currently-configured account cannot be + // verified: if the store has since been repointed, DeleteIfExistsAsync returns false and the purge + // would silently report success while the real blob survives in the old account. Rather than delete + // on an unverifiable pointer, the token is discarded. The backend ack protocol carries no "skip" + // status (PayloadPurgeAck is just partition/instance/payload id, and an un-acked row is re-served by + // an uncursored TOP(N) query every cycle), so declining without acking would permanently block the + // pipeline. The full token is logged at error level so it remains a recoverable pointer. + this.logger.BlobPurgeDeleteV1TokenUnsupported(input); + return BlobDeleteResult.Discarded; + } + try { await this.store.DeleteAsync(input, CancellationToken.None); @@ -75,6 +97,16 @@ public override async Task RunAsync(TaskActivityContext contex this.logger.BlobPurgeDeleteDiscarded(ex, input); return BlobDeleteResult.Discarded; } + catch (NotSupportedException ex) + { + // The registered store does not implement deletion (PayloadStore.DeleteAsync is virtual and its base + // implementation throws). This is a misconfiguration, not poison data: every payload would fail the + // same way, so acking would hard-delete the backend's entire record of what still needs cleanup while + // every blob survives. Keep the payload tombstoned so the work is recoverable once an operator + // registers a store that can delete. + this.logger.BlobPurgeDeleteNotSupported(ex, input); + return BlobDeleteResult.Retry; + } catch (PayloadStorageException ex) { // The token is well-formed but points at a storage account this worker's credential cannot reach diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index bc3e18a5..ed8ac7d7 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -14,7 +14,7 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// Client-side hosted service that ensures the singleton blob payload auto-purge job exists. It is registered /// only when auto-purge is enabled at registration time (see the UseExternalizedPayloads configure overload), /// so it does not re-check the flag here. It never blocks host startup: it runs on a background task and -/// retries until the backend is reachable. The job is a whole-scheduler singleton, so racing client processes +/// retries until the backend is reachable. The job is a per-task-hub singleton, so racing client processes /// simply no-op. /// sealed class BlobPurgeJobStarter : IHostedService @@ -22,6 +22,7 @@ sealed class BlobPurgeJobStarter : IHostedService static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); readonly DurableTaskClient client; + readonly PayloadStore store; readonly IOptionsMonitor options; readonly string builderName; readonly ILogger logger; @@ -32,11 +33,13 @@ sealed class BlobPurgeJobStarter : IHostedService public BlobPurgeJobStarter( DurableTaskClient client, + PayloadStore store, IOptionsMonitor options, string builderName, ILogger logger) { this.client = Check.NotNull(client); + this.store = Check.NotNull(store); this.options = Check.NotNull(options); this.builderName = Check.NotNull(builderName); this.logger = Check.NotNull(logger); @@ -45,6 +48,17 @@ public BlobPurgeJobStarter( /// public Task StartAsync(CancellationToken cancellationToken) { + // Auto-purge deletes blobs through the store, but PayloadStore.DeleteAsync is virtual and its base + // implementation throws NotSupportedException. A store that cannot delete would fail every single + // payload, so refuse to start the job rather than spin against the backend - and rather than ack rows + // whose blobs were never deleted, which would destroy the backend's record of what still needs cleanup. + // This is a configuration error and is surfaced at startup, where it is cheapest to notice. + if (this.store is not BlobPayloadStore) + { + this.logger.BlobPurgeStoreCannotDelete(this.store.GetType().FullName); + return Task.CompletedTask; + } + LargePayloadStorageOptions opts = this.options.Get(this.builderName); int batchSize = opts.PayloadPurgeBatchSize; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 89391890..2d55a979 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -13,7 +13,7 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; class BlobPurgeJob(ILogger logger) : TaskEntity { /// - /// Creates (or reactivates) the auto-purge job. Because the job is a whole-scheduler singleton, this is + /// Creates (or reactivates) the auto-purge job. Because the job is a per-task-hub singleton, this is /// intentionally a no-op when the job is already so that extra /// client processes racing to create it do not disturb the running job. /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 5238496d..a84a2ff8 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -42,4 +42,13 @@ static partial class Logs [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' points at a storage account the configured credential cannot reach; the blob cannot be deleted by this worker and will be orphaned. Acknowledging it so the backend can clear the row - reclaim the blob manually or reconfigure the payload store with identity (AAD) authentication that can access both accounts.")] public static partial void BlobPurgeDeleteUnreachable(this ILogger logger, Exception exception, string token); + + [LoggerMessage(EventId = 822, Level = LogLevel.Error, Message = "Externalized payload token '{token}' uses the legacy v1 format, which auto-purge does not delete because it does not identify the storage account. The backing blob will NOT be deleted and is now orphaned; the row is acknowledged so the purge pipeline is not blocked. Reclaim the blob using the container and blob name in the token above, and upgrade to an SDK version that writes self-describing v2 tokens.")] + public static partial void BlobPurgeDeleteV1TokenUnsupported(this ILogger logger, string token); + + [LoggerMessage(EventId = 823, Level = LogLevel.Error, Message = "Blob payload auto-purge is enabled but the registered PayloadStore ('{storeType}') is not an Azure Blob payload store and cannot delete payloads. The auto-purge job was not started; externalized payloads will not be reclaimed. Register the Azure Blob payload store, or disable AutoPurge.")] + public static partial void BlobPurgeStoreCannotDelete(this ILogger logger, string? storeType); + + [LoggerMessage(EventId = 824, Level = LogLevel.Error, Message = "The registered PayloadStore does not support deleting payloads, so externalized payload token '{token}' cannot be purged. Leaving it tombstoned; register an Azure Blob payload store on the worker or disable AutoPurge.")] + public static partial void BlobPurgeDeleteNotSupported(this ILogger logger, Exception exception, string token); } diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 0c485208..e924cb3e 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -106,6 +106,7 @@ static void RegisterBlobPurgeJobStarter(IDurableTaskClientBuilder builder) string builderName = builder.Name; builder.Services.AddSingleton(sp => new BlobPurgeJobStarter( sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetRequiredService>(), builderName, sp.GetRequiredService>())); diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 0edaee90..9339c997 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -124,6 +124,12 @@ public int ThresholdBytes /// the corresponding blobs from customer storage (the backend has no storage credentials of its own). /// Defaults to false (opt-in). /// + /// + /// Auto-purge only deletes blobs referenced by self-describing blob:v2: tokens. Payloads that were + /// externalized by SDK versions that wrote legacy blob:v1: tokens are acknowledged and logged at + /// error level, but their blobs are not deleted: a v1 token does not identify the storage account, so the + /// delete cannot be verified. + /// public bool AutoPurge { get; set; } /// diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index 7a9cbacb..ca73812f 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -23,7 +23,7 @@ namespace Microsoft.DurableTask; Justification = "SemaphoreSlim does not allocate a disposable resource unless AvailableWaitHandle is accessed.")] public sealed class BlobPayloadStore : PayloadStore { - const string TokenPrefixV1 = "blob:v1:"; + internal const string TokenPrefixV1 = "blob:v1:"; const string TokenPrefixV2 = "blob:v2:"; const string ContentEncodingGzip = "gzip"; const int MaxRetryAttempts = 8; diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs new file mode 100644 index 00000000..e8617b2b --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class BlobPurgeJobStarterTests +{ + [Fact] + public async Task StartAsync_WhenStoreCannotDelete_DoesNotStartJobOrTouchClient() + { + // Arrange - the registered store is not the blob store, so its DeleteAsync is unsupported. Every payload + // would fail, so the starter must refuse to run rather than spin against the backend. + Mock client = new("test"); + BlobPurgeJobStarter starter = new( + client.Object, + new NonDeletingPayloadStore(), + OptionsFor(new LargePayloadStorageOptions { AutoPurge = true }), + "test", + new TestLogger()); + + // Act + await starter.StartAsync(CancellationToken.None); + + // Assert - the startup gate short-circuits before scheduling anything, so the client is never queried. + client.Verify( + c => c.GetInstanceAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task StartAsync_WhenStoreIsBlobStore_DoesNotShortCircuit() + { + // Arrange - the real blob store. UseDevelopmentStorage=true is a valid connection string that constructs + // the store offline (no network I/O), so resolving it at host start is safe. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + Mock client = new("test"); + TestLogger logger = new(); + BlobPurgeJobStarter starter = new( + client.Object, + store, + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true")), + "test", + logger); + + // Act + await starter.StartAsync(CancellationToken.None); + + // Assert - the store-cannot-delete gate did NOT fire; the starter proceeded to its background ensure + // path (which runs on a background task, so timing is not asserted). + logger.Logs.Should().NotContain(entry => entry.Message.Contains("is not an Azure Blob payload store")); + + // Cleanup - cancel the background ensure task so it does not outlive the test. + await starter.StopAsync(CancellationToken.None); + } + + static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) + { + Mock> monitor = new(); + monitor.Setup(m => m.Get(It.IsAny())).Returns(options); + return monitor.Object; + } + + sealed class NonDeletingPayloadStore : PayloadStore + { + // DeleteAsync is intentionally NOT overridden: the base PayloadStore.DeleteAsync throws + // NotSupportedException, which is exactly the "store cannot delete" configuration the starter refuses. + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => false; + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 5e3004cb..93989325 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -18,7 +18,7 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailed400_DiscardsPoisonToken( DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:bad name"); + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/bad name"); // Assert - discarded so the backend acks and clears the row instead of re-streaming forever. result.Should().Be(BlobDeleteResult.Discarded); @@ -32,7 +32,7 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_LeavesTombstonedF DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); // Assert - left tombstoned so a later purge cycle can retry; a blob is never dropped on doubt. result.Should().Be(BlobDeleteResult.Retry); @@ -54,6 +54,54 @@ public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_DiscardsToUnb result.Should().Be(BlobDeleteResult.Discarded); } + [Fact] + public async Task RunAsync_V1Token_DiscardsWithoutCallingStore() + { + // Arrange - auto-purge policy: a legacy v1 token identifies no storage account, so it is dropped before + // the store is ever consulted. + Mock store = new(); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + + // Assert - discarded by the gate, and the store's DeleteAsync was never invoked. + result.Should().Be(BlobDeleteResult.Discarded); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsync_V2Token_CallsStore() + { + // Arrange - a self-describing v2 token is not gated and must reach the store. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())).Returns(Task.CompletedTask); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + + // Assert - the store deleted the blob (proves the gate is v1-only and did not break the happy path). + result.Should().Be(BlobDeleteResult.Deleted); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombstone() + { + // Arrange - a store that cannot delete (the base PayloadStore.DeleteAsync throws NotSupportedException). + StubPayloadStore store = new(new NotSupportedException()); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + + // Assert - retried (tombstone preserved), never discarded: acking would destroy the backend's cleanup + // ledger while the blob survives. + result.Should().Be(BlobDeleteResult.Retry); + result.Should().NotBe(BlobDeleteResult.Discarded); + } + sealed class StubPayloadStore : PayloadStore { readonly Exception? deleteError; From 65e9cbb1c5ca1c773fcba0e17e02f7f61605cfc4 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 14:53:05 -0700 Subject: [PATCH 15/47] Fix auto-purge starter registration and client resolution Address three reviewer findings on the blob payload auto-purge starter: - Finding 1: register BlobPurgeJobStarter unconditionally from UseExternalizedPayloadsCore (both overloads) instead of probing the configure delegate at registration time. The probe double-invoked user code and missed every enable path except the inline delegate (services.Configure, config binding, PostConfigure, parameterless overload). The AutoPurge decision now happens in StartAsync, once options are fully resolved, and no-ops silently when disabled. - Finding 2: inject IDurableTaskClientProvider and resolve the client by builder name lazily in StartAsync (after both gates) so named clients get their own client and no DurableTaskClient is constructed at host start for apps that externalize payloads without auto-purge. - Finding 3: pass the purge retry policy to the delete activity call in BlobPurgeJobOrchestrator.DeleteOneAsync, matching the other two calls. Tests: update starter tests for the new ctor, add a disabled-path test asserting no client resolution or logging, and add DI regression tests covering the services.Configure enable path and single configure invocation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 44 +++++++++----- .../BlobPurgeJobOrchestrator.cs | 3 +- ...ientBuilderExtensions.AzureBlobPayloads.cs | 21 +++---- .../AutoPurge/BlobPurgeJobStarterTests.cs | 54 +++++++++++++----- .../UseExternalizedPayloadsTests.cs | 57 ++++++++++++++++--- 5 files changed, 131 insertions(+), 48 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index ed8ac7d7..b6f10d9c 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -11,17 +11,18 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Client-side hosted service that ensures the singleton blob payload auto-purge job exists. It is registered -/// only when auto-purge is enabled at registration time (see the UseExternalizedPayloads configure overload), -/// so it does not re-check the flag here. It never blocks host startup: it runs on a background task and -/// retries until the backend is reachable. The job is a per-task-hub singleton, so racing client processes -/// simply no-op. +/// Client-side hosted service that ensures the per-task-hub singleton blob payload auto-purge job exists. It is +/// registered unconditionally by UseExternalizedPayloads and decides what to do at startup, once options are +/// fully resolved: it no-ops silently when auto-purge is disabled, and no-ops with an error log when the +/// registered store cannot delete. It never blocks host startup - the ensure work runs on a background task +/// that retries until the backend is reachable. The job is a per-task-hub singleton, so racing client +/// processes simply no-op. /// sealed class BlobPurgeJobStarter : IHostedService { static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); - readonly DurableTaskClient client; + readonly IDurableTaskClientProvider clientProvider; readonly PayloadStore store; readonly IOptionsMonitor options; readonly string builderName; @@ -32,13 +33,13 @@ sealed class BlobPurgeJobStarter : IHostedService Task? ensureTask; public BlobPurgeJobStarter( - DurableTaskClient client, + IDurableTaskClientProvider clientProvider, PayloadStore store, IOptionsMonitor options, string builderName, ILogger logger) { - this.client = Check.NotNull(client); + this.clientProvider = Check.NotNull(clientProvider); this.store = Check.NotNull(store); this.options = Check.NotNull(options); this.builderName = Check.NotNull(builderName); @@ -48,6 +49,19 @@ public BlobPurgeJobStarter( /// public Task StartAsync(CancellationToken cancellationToken) { + LargePayloadStorageOptions opts = this.options.Get(this.builderName); + + // Not opted in. The starter is registered unconditionally by UseExternalizedPayloads because whether + // auto-purge is enabled can only be known once options are fully resolved - the flag can be set by the + // inline configure delegate, services.Configure, configuration binding or PostConfigure. Deciding at + // registration time (by running the delegate against a probe instance) both invoked user code twice and + // missed every enable path except the inline delegate. This is the normal path for apps that externalize + // payloads without auto-purge, so it returns silently without logging. + if (!opts.AutoPurge) + { + return Task.CompletedTask; + } + // Auto-purge deletes blobs through the store, but PayloadStore.DeleteAsync is virtual and its base // implementation throws NotSupportedException. A store that cannot delete would fail every single // payload, so refuse to start the job rather than spin against the backend - and rather than ack rows @@ -59,13 +73,17 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - LargePayloadStorageOptions opts = this.options.Get(this.builderName); + // Resolve the client by builder name rather than by type: a named client builder must get its own + // client, and resolving lazily here - after the AutoPurge gate - avoids constructing a DurableTaskClient + // at host start for apps that externalize payloads without auto-purge. + DurableTaskClient client = this.clientProvider.GetClient(this.builderName); + int batchSize = opts.PayloadPurgeBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend // is reachable. this.cts = new CancellationTokenSource(); - this.ensureTask = Task.Run(() => this.EnsureJobAsync(batchSize, this.cts.Token), CancellationToken.None); + this.ensureTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); return Task.CompletedTask; } @@ -82,7 +100,7 @@ public async Task StopAsync(CancellationToken cancellationToken) } } - async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) + async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { @@ -96,7 +114,7 @@ async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) // id and no dedupe policy the backend would purge and replace the terminal instance on every // host restart.) Only (re)schedule when the bridge is absent, or ended in a Failed/Terminated // state that may never have applied Create - which lets a failed setup self-heal. - OrchestrationMetadata? existing = await this.client.GetInstanceAsync( + OrchestrationMetadata? existing = await client.GetInstanceAsync( BlobPurgeConstants.StarterInstanceId, cancellationToken); bool needsSchedule = existing is null @@ -110,7 +128,7 @@ async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) BlobPurgeJobOperationRequest request = new( this.entityId, nameof(BlobPurgeJob.Create), batchSize); - await this.client.ScheduleNewOrchestrationInstanceAsync( + await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId), diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 8ba4f707..6523001b 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -145,7 +145,8 @@ async Task DeleteOneAsync(TaskOrchestrationContext context, Tombs { BlobDeleteResult result = await context.CallActivityAsync( nameof(DeleteExternalBlobActivity), - tombstone.Token); + tombstone.Token, + new TaskOptions(PurgeActivityRetryPolicy)); return new DeleteOutcome( result != BlobDeleteResult.Retry, diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index e924cb3e..f54d499b 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -35,18 +35,7 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( builder.Services.Configure(builder.Name, configure); - UseExternalizedPayloadsCore(builder); - - // Conditional DI: register the auto-purge starter only when the caller opted into auto-purge. Peek the - // flag now by running the configure delegate against a probe (options configurators are pure setters). - LargePayloadStorageOptions probe = new(); - configure(probe); - if (probe.AutoPurge) - { - RegisterBlobPurgeJobStarter(builder); - } - - return builder; + return UseExternalizedPayloadsCore(builder); } /// @@ -98,6 +87,12 @@ static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientB } }); + // Always register the auto-purge starter. Whether auto-purge is actually enabled can only be known once + // options are fully resolved - the flag can be set by the inline configure delegate, services.Configure, + // configuration binding or PostConfigure, none of which are visible here at registration time - so the + // starter is registered unconditionally and no-ops in StartAsync when AutoPurge is disabled. + RegisterBlobPurgeJobStarter(builder); + return builder; } @@ -105,7 +100,7 @@ static void RegisterBlobPurgeJobStarter(IDurableTaskClientBuilder builder) { string builderName = builder.Name; builder.Services.AddSingleton(sp => new BlobPurgeJobStarter( - sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService>(), builderName, diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index e8617b2b..f79c79e6 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -15,10 +15,11 @@ public class BlobPurgeJobStarterTests public async Task StartAsync_WhenStoreCannotDelete_DoesNotStartJobOrTouchClient() { // Arrange - the registered store is not the blob store, so its DeleteAsync is unsupported. Every payload - // would fail, so the starter must refuse to run rather than spin against the backend. - Mock client = new("test"); + // would fail, so the starter must refuse to run rather than spin against the backend. AutoPurge is on so + // the run gets past the opt-in gate and reaches the store-capability gate under test. + Mock provider = new(); BlobPurgeJobStarter starter = new( - client.Object, + provider.Object, new NonDeletingPayloadStore(), OptionsFor(new LargePayloadStorageOptions { AutoPurge = true }), "test", @@ -27,37 +28,64 @@ public async Task StartAsync_WhenStoreCannotDelete_DoesNotStartJobOrTouchClient( // Act await starter.StartAsync(CancellationToken.None); - // Assert - the startup gate short-circuits before scheduling anything, so the client is never queried. - client.Verify( - c => c.GetInstanceAsync(It.IsAny(), It.IsAny()), Times.Never); + // Assert - the store gate short-circuits before the client is resolved, so the provider is never asked + // for a client. + provider.Verify(p => p.GetClient(It.IsAny()), Times.Never); } [Fact] public async Task StartAsync_WhenStoreIsBlobStore_DoesNotShortCircuit() { - // Arrange - the real blob store. UseDevelopmentStorage=true is a valid connection string that constructs - // the store offline (no network I/O), so resolving it at host start is safe. + // Arrange - the real blob store with auto-purge enabled. UseDevelopmentStorage=true is a valid connection + // string that constructs the store offline (no network I/O), so resolving it at host start is safe. BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); - Mock client = new("test"); + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(new Mock("test").Object); TestLogger logger = new(); BlobPurgeJobStarter starter = new( - client.Object, + provider.Object, store, - OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true")), + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = true }), "test", logger); // Act await starter.StartAsync(CancellationToken.None); - // Assert - the store-cannot-delete gate did NOT fire; the starter proceeded to its background ensure - // path (which runs on a background task, so timing is not asserted). + // Assert - neither gate fired: no store-cannot-delete log, and the starter proceeded to resolve the + // client and start its background ensure path (which runs on a background task, so timing is not + // asserted). logger.Logs.Should().NotContain(entry => entry.Message.Contains("is not an Azure Blob payload store")); + provider.Verify(p => p.GetClient(It.IsAny()), Times.Once); // Cleanup - cancel the background ensure task so it does not outlive the test. await starter.StopAsync(CancellationToken.None); } + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabled_DoesNotResolveClientOrLog() + { + // Arrange - auto-purge is off. Even with a delete-capable blob store, the starter must no-op silently: + // it is registered unconditionally, so the not-opted-in path is the common case and must not log or + // resolve a client. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + Mock provider = new(); + TestLogger logger = new(); + BlobPurgeJobStarter starter = new( + provider.Object, + store, + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = false }), + "test", + logger); + + // Act + await starter.StartAsync(CancellationToken.None); + + // Assert - returned before resolving a client and without logging anything at all. + provider.Verify(p => p.GetClient(It.IsAny()), Times.Never); + logger.Logs.Should().BeEmpty(); + } + static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) { Mock> monitor = new(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index 2ee439ef..05013de4 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -2,9 +2,13 @@ // Licensed under the MIT License. using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; using Moq; using Xunit; @@ -29,7 +33,7 @@ public void UseExternalizedPayloads_WithAutoPurgeEnabled_RegistersHostedPurgeSta } [Fact] - public void UseExternalizedPayloads_WithAutoPurgeDisabled_DoesNotRegisterHostedPurgeStarter() + public void UseExternalizedPayloads_WithAutoPurgeDisabled_StillRegistersHostedPurgeStarter() { // Arrange ServiceCollection services = new(); @@ -40,24 +44,61 @@ public void UseExternalizedPayloads_WithAutoPurgeDisabled_DoesNotRegisterHostedP // Act - auto-purge left at its default (false). builder.Object.UseExternalizedPayloads(options => { }); - // Assert - services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + // Assert - the starter is registered unconditionally now; whether auto-purge is enabled can only be + // known once options are fully resolved, so the no-op moved into BlobPurgeJobStarter.StartAsync. Do not + // "fix" this back to NotContain - the registration is intentional. + services.Should().ContainSingle(d => d.ServiceType == typeof(IHostedService)); } [Fact] - public void UseExternalizedPayloads_NoArgOverload_DoesNotRegisterHostedPurgeStarter() + public void UseExternalizedPayloads_AutoPurgeViaServicesConfigure_RegistersResolvableStarter() { - // Arrange + // Arrange - the enable path that silently no-op'd before: AutoPurge set through services.Configure (not + // the inline delegate) plus the parameterless overload. The old probe-at-registration only saw the + // inline delegate, so the starter was never registered. It must now be registered and, more importantly, + // resolvable from the built provider. ServiceCollection services = new(); + services.AddSingleton>(NullLogger.Instance); + services.AddSingleton(Mock.Of()); + services.Configure(o => + { + o.AutoPurge = true; + o.ConnectionString = "UseDevelopmentStorage=true"; + }); + Mock builder = new(); builder.Setup(b => b.Services).Returns(services); builder.Setup(b => b.Name).Returns(string.Empty); - // Act - the shared-store overload never enables auto-purge. + // Act - the parameterless overload; AutoPurge comes from services.Configure above. builder.Object.UseExternalizedPayloads(); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - the starter resolves as a hosted service (construction pulls PayloadStore, the client + // provider, options and logger), proving the whole enable path is wired end to end. + provider.GetServices().OfType().Should().ContainSingle(); + } + + [Fact] + public void UseExternalizedPayloads_ConfigureDelegate_InvokedExactlyOnce() + { + // Arrange - a delegate that counts its invocations. The old probe-at-registration ran configure a second + // time against a throwaway options instance; this locks in that user code runs exactly once, when the + // named options are first materialized. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + int invocations = 0; + + // Act + builder.Object.UseExternalizedPayloads(options => invocations++); + using ServiceProvider provider = services.BuildServiceProvider(); + provider.GetRequiredService>().Get(string.Empty); - // Assert - services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + // Assert - configure ran once (at options materialization), not a second time at registration. + invocations.Should().Be(1); } [Fact] From 8f436df07a535fdc1c5f0336f5a6fd7926227e5f Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 15:02:24 -0700 Subject: [PATCH 16/47] Back off on zero-ack purge cycles and document TokenPrefixV1 Item A: add the SA1600 doc comment to the internal TokenPrefixV1 const in BlobPayloadStore.cs, restoring the multi-TFM warning baseline to 201. Item B: add an else branch to BlobPurgeJobOrchestrator.RunAsync for the case where a batch produced no acks. DeleteExternalBlobActivity reports storage failures as a BlobDeleteResult.Retry return value rather than an exception, so the activity retry policy never engages and there is no backoff on that path. Combined with the backend's uncursored TOP(N) tombstone query, an immediate continue would refetch the identical rows and re-attempt the identical deletes in a tight loop for the duration of a storage outage. Back off on ErrorBackoff (the existing one-minute timer) before the next cycle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../Orchestrations/BlobPurgeJobOrchestrator.cs | 10 ++++++++++ .../AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 6523001b..50758e60 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -90,6 +90,16 @@ await context.CallActivityAsync( await context.Entities.CallEntityAsync( input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)acks.Count); } + else + { + // Nothing in this batch could be acknowledged: every delete returned Retry (e.g. a storage + // outage or throttling). Deletes report failure as a return value rather than an exception, + // so no retry policy or backoff applies on that path. The backend serves tombstones with an + // uncursored TOP(N) query, so continuing immediately would refetch the identical rows and + // re-attempt the identical deletes in a tight loop for as long as the outage lasts. Back off + // before trying again. + await context.CreateTimer(ErrorBackoff, default); + } } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index ca73812f..d67f967c 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -23,6 +23,10 @@ namespace Microsoft.DurableTask; Justification = "SemaphoreSlim does not allocate a disposable resource unless AvailableWaitHandle is accessed.")] public sealed class BlobPayloadStore : PayloadStore { + /// + /// The prefix of legacy v1 payload tokens, which identify the container by name only and not the storage + /// account. Auto-purge uses this to detect and skip v1 tokens. + /// internal const string TokenPrefixV1 = "blob:v1:"; const string TokenPrefixV2 = "blob:v2:"; const string ContentEncodingGzip = "gzip"; From 81284e025cd9336bc946e0819fc5253bbc33c287 Mon Sep 17 00:00:00 2001 From: wangbill Date: Sun, 2 Aug 2026 12:33:47 -0700 Subject: [PATCH 17/47] docs(AzureBlobPayloads): reframe v1-token handling as a defensive guard The DTS backend now hard-deletes legacy blob:v1: payload rows instead of tombstoning them, so the SDK will normally never be handed a v1 token. Update the prose that described v1-tombstoning as the expected path. - LargePayloadStorageOptions.AutoPurge (public API): v1-backed blobs are not reclaimed and remain in storage exactly as before auto-purge existed (not a new leak); the backend removes their rows normally. - Logs.cs EventId 822: reworded to read as unexpected (older backend, or a row tombstoned before the fix); keeps Error level, EventId 822, and the full token. - DeleteExternalBlobActivity item + inline gate comment: note the v1 branch is now a defensive guard for backend version skew, not the expected path. Prose only; no behavioural change. The only non-comment edit is the 822 message string. Multi-TFM warning baseline unchanged (201); AzureBlobPayloads tests 42/42. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../AutoPurge/Activities/DeleteExternalBlobActivity.cs | 8 ++++++-- src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs | 2 +- .../Options/LargePayloadStorageOptions.cs | 10 ++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 12438d47..e90c5543 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -24,7 +24,9 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// name, not the storage account, so a delete against the currently-configured account cannot be verified - if /// the store has been repointed the delete would report success while the real blob survives elsewhere. The /// backend ack protocol has no "skip" status (an un-acked row is re-served every cycle), so the token is acked -/// to keep the pipeline moving and logged at error level as the operator's recovery pointer. +/// to keep the pipeline moving and logged at error level as the operator's recovery pointer. A current backend +/// hard-deletes v1 rows instead of tombstoning them, so this branch is a defensive guard for an older backend +/// build or a row tombstoned before that fix, not the expected path. /// /// /// Permanent failures are discarded (acked so the backend clears the row) because retrying can never succeed: @@ -72,7 +74,9 @@ public override async Task RunAsync(TaskActivityContext contex // on an unverifiable pointer, the token is discarded. The backend ack protocol carries no "skip" // status (PayloadPurgeAck is just partition/instance/payload id, and an un-acked row is re-served by // an uncursored TOP(N) query every cycle), so declining without acking would permanently block the - // pipeline. The full token is logged at error level so it remains a recoverable pointer. + // pipeline. The full token is logged at error level so it remains a recoverable pointer. A current + // backend hard-deletes v1 rows instead of tombstoning them, so reaching this branch means an older + // backend build or a row tombstoned before that fix - it is a defensive guard, not the expected path. this.logger.BlobPurgeDeleteV1TokenUnsupported(input); return BlobDeleteResult.Discarded; } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index a84a2ff8..f2989b9f 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -43,7 +43,7 @@ static partial class Logs [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' points at a storage account the configured credential cannot reach; the blob cannot be deleted by this worker and will be orphaned. Acknowledging it so the backend can clear the row - reclaim the blob manually or reconfigure the payload store with identity (AAD) authentication that can access both accounts.")] public static partial void BlobPurgeDeleteUnreachable(this ILogger logger, Exception exception, string token); - [LoggerMessage(EventId = 822, Level = LogLevel.Error, Message = "Externalized payload token '{token}' uses the legacy v1 format, which auto-purge does not delete because it does not identify the storage account. The backing blob will NOT be deleted and is now orphaned; the row is acknowledged so the purge pipeline is not blocked. Reclaim the blob using the container and blob name in the token above, and upgrade to an SDK version that writes self-describing v2 tokens.")] + [LoggerMessage(EventId = 822, Level = LogLevel.Error, Message = "Received a legacy v1 externalized payload token '{token}' from the backend, which is unexpected: a current backend hard-deletes v1 payload rows instead of tombstoning them, so this indicates either an older backend build or a row that was tombstoned before that fix. Auto-purge does not delete v1 tokens because a v1 token identifies the container by name only and not the storage account, so the delete cannot be verified; the backing blob is NOT deleted and the row is acknowledged so the purge pipeline is not blocked. Reclaim the blob using the container and blob name in the token above, and upgrade to an SDK version that writes self-describing v2 tokens.")] public static partial void BlobPurgeDeleteV1TokenUnsupported(this ILogger logger, string token); [LoggerMessage(EventId = 823, Level = LogLevel.Error, Message = "Blob payload auto-purge is enabled but the registered PayloadStore ('{storeType}') is not an Azure Blob payload store and cannot delete payloads. The auto-purge job was not started; externalized payloads will not be reclaimed. Register the Azure Blob payload store, or disable AutoPurge.")] diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 9339c997..2b0b807a 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -125,10 +125,12 @@ public int ThresholdBytes /// Defaults to false (opt-in). /// /// - /// Auto-purge only deletes blobs referenced by self-describing blob:v2: tokens. Payloads that were - /// externalized by SDK versions that wrote legacy blob:v1: tokens are acknowledged and logged at - /// error level, but their blobs are not deleted: a v1 token does not identify the storage account, so the - /// delete cannot be verified. + /// Auto-purge reclaims only blobs referenced by self-describing blob:v2: tokens. Payloads written by + /// SDK versions that emitted legacy blob:v1: tokens are not reclaimed, because a v1 token identifies + /// the container by name only and not the storage account, so the delete cannot be verified. Their backing + /// blobs remain in storage exactly as they did before auto-purge existed - this is not a new leak introduced + /// by auto-purge - and the backend removes their rows normally. Upgrading to an SDK version that writes v2 + /// tokens makes newly-externalized payloads eligible. /// public bool AutoPurge { get; set; } From 1fcdb1061f219ae254b7c47a63f9054f36e96c60 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 15:15:25 -0700 Subject: [PATCH 18/47] Reshape large-payload auto-purge to the finalized design Replaces the earlier draft shape with the finalized contract and dispositions. Wire contract (orchestrator_service.proto): drops GetTombstonedPayloads / AckPurgedPayloads and their messages, and splices in the canonical block verbatim -- GetLargePayloadTombstones / ReportLargePayloadPurgeResults, the LargePayloadPurgeDisposition and LargePayloadPurgeReason enums, and large_payload_auto_purge_enabled = 12 on GetWorkItemsRequest. Dispositions: removes Discarded entirely. It was success-shaped and destroyed evidence. Every attempt now returns a disposition plus a stable reason code. v1 tokens and malformed v2 bodies are Quarantined; unknown version prefixes, unreachable accounts, and authorization failures are Retry. Splitting the store's three ArgumentException cases is done with a token prefix gate in the activity rather than new exception types, which required widening TokenPrefixV2 to internal. Ownership marker: uploads now stamp managed_by=dts as blob metadata on both the compressed and uncompressed write paths, and the delete path reads the blob's properties and passes that read's ETag as an If-Match on the delete, so check-and-delete is atomic without a lease. A blob without the marker is left untouched and reported DELETED / BLOB_NOT_STORE_OWNED. Metadata rather than index tags: tags are unsupported on ADLS Gen2 and would require Storage Blob Data Owner. Orchestration: the batch is now always reported, since the backend owns retry scheduling and needs to hear about failures to defer a row. Backoff is preserved for the all-retry case so a storage outage cannot become a tight refetch loop. Progress counts only Deleted rows; counting quarantined rows as purged would reintroduce the success-shaped reporting this change removes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Client/Core/DurableTaskClient.cs | 26 +-- .../Core/LargePayloadPurgeDisposition.cs | 33 +++ src/Client/Core/LargePayloadPurgeReason.cs | 91 ++++++++ src/Client/Core/LargePayloadPurgeResult.cs | 34 +++ src/Client/Core/LargePayloadTombstone.cs | 23 ++ src/Client/Core/PayloadPurgeAck.cs | 13 -- src/Client/Core/TombstonedPayload.cs | 15 -- src/Client/Grpc/GrpcDurableTaskClient.cs | 53 +++-- .../Activities/AckPurgedPayloadsActivity.cs | 36 --- .../Activities/DeleteExternalBlobActivity.cs | 213 +++++++++++------- .../GetLargePayloadTombstonesActivity.cs | 33 +++ .../GetTombstonedPayloadsActivity.cs | 32 --- .../ReportLargePayloadPurgeResultsActivity.cs | 39 ++++ .../AutoPurge/Constants/BlobPurgeConstants.cs | 2 +- .../AzureBlobPayloads/AutoPurge/Logs.cs | 22 +- .../AutoPurge/Models/BlobDeleteResult.cs | 27 --- .../AutoPurge/Models/BlobPurgeOutcome.cs | 22 ++ .../BlobPurgeJobOrchestrator.cs | 122 ++++++---- ...rkerBuilderExtensions.AzureBlobPayloads.cs | 11 +- .../PayloadStore/BlobPayloadStore.cs | 103 ++++++++- .../PayloadStore/PayloadDeleteOutcome.cs | 28 +++ .../PayloadStore/PayloadStore.cs | 10 +- src/Grpc/orchestrator_service.proto | 156 ++++++++++--- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 4 + .../Grpc/GrpcDurableTaskWorkerOptions.cs | 7 + .../LargePayloadPurgeEnumParityTests.cs | 47 ++++ .../DeleteExternalBlobActivityTests.cs | 185 ++++++++++++--- .../BlobPayloadStoreTests.cs | 42 ++++ .../UseExternalizedPayloadsTests.cs | 37 ++- .../BlobPayloadStoreDeleteTests.cs | 83 ++++++- 30 files changed, 1165 insertions(+), 384 deletions(-) create mode 100644 src/Client/Core/LargePayloadPurgeDisposition.cs create mode 100644 src/Client/Core/LargePayloadPurgeReason.cs create mode 100644 src/Client/Core/LargePayloadPurgeResult.cs create mode 100644 src/Client/Core/LargePayloadTombstone.cs delete mode 100644 src/Client/Core/PayloadPurgeAck.cs delete mode 100644 src/Client/Core/TombstonedPayload.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs create mode 100644 src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs create mode 100644 test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index bc31915c..c903574a 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -550,28 +550,28 @@ public virtual Task> ListInstanceIdsAsync( } /// - /// Gets a batch of tombstoned (soft-deleted) externalized payloads whose backing blobs should be deleted - /// by a credentialed caller before the backend hard-deletes the rows. + /// Gets a batch of due large-payload tombstones whose backing blobs a credentialed caller must delete. /// - /// The maximum number of tombstoned payloads to request. + /// The maximum number of tombstones to request. /// The cancellation token. - /// The batch of tombstoned payloads whose blobs should be deleted. + /// The batch of tombstones whose blobs should be deleted. /// Thrown if this implementation does not support the operation. - public virtual Task> GetTombstonedPayloadsAsync( + public virtual Task> GetLargePayloadTombstonesAsync( int limit, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support retrieving tombstoned payloads."); + => throw new NotSupportedException($"{this.GetType()} does not support retrieving large-payload tombstones."); /// - /// Acknowledges tombstoned payloads whose backing blobs have been deleted so the backend can hard-delete - /// the corresponding rows. + /// Reports the outcome of each attempted large-payload blob deletion so the backend can resolve, + /// reschedule, or quarantine the corresponding tombstones. Every attempted row is reported, not just the + /// successful ones; the backend owns retry scheduling. /// - /// The payloads whose blobs have been deleted. + /// The per-row outcomes of the attempted deletions. /// The cancellation token. - /// A task that completes when the acknowledgement has been sent. + /// A task that completes when the outcomes have been recorded. /// Thrown if this implementation does not support the operation. - public virtual Task AckPurgedPayloadsAsync( - IEnumerable acks, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support acknowledging purged payloads."); + public virtual Task ReportLargePayloadPurgeResultsAsync( + IEnumerable results, CancellationToken cancellation = default) + => throw new NotSupportedException($"{this.GetType()} does not support reporting large-payload purge results."); // TODO: Create task hub diff --git a/src/Client/Core/LargePayloadPurgeDisposition.cs b/src/Client/Core/LargePayloadPurgeDisposition.cs new file mode 100644 index 00000000..bfdedca2 --- /dev/null +++ b/src/Client/Core/LargePayloadPurgeDisposition.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// The outcome of a single large-payload blob deletion attempt. The split is by whether a failure can +/// self-heal. Mirrors the LargePayloadPurgeDisposition protobuf enum. +/// +public enum LargePayloadPurgeDisposition +{ + /// + /// No disposition was specified. + /// + Unspecified = 0, + + /// + /// Terminal success. The blob was deleted, was already absent, or was deliberately left in place because + /// it is not owned by the payload store. The backend deletes the tombstone in all three cases. + /// + Deleted = 1, + + /// + /// The failure may self-heal, so the row stays pending and the backend sets the next attempt. + /// + Retry = 2, + + /// + /// A deterministic failure or protocol violation that retrying can never fix. The backend preserves the + /// evidence, alerts, and stops automatic retries. + /// + Quarantined = 3, +} diff --git a/src/Client/Core/LargePayloadPurgeReason.cs b/src/Client/Core/LargePayloadPurgeReason.cs new file mode 100644 index 00000000..4be5190d --- /dev/null +++ b/src/Client/Core/LargePayloadPurgeReason.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// A stable, bounded reason for a . Never carries a token or raw +/// exception text, because tokens expose the storage account, container, and blob path. Mirrors the +/// LargePayloadPurgeReason protobuf enum. +/// +public enum LargePayloadPurgeReason +{ + /// + /// No reason was specified. + /// + Unspecified = 0, + + /// + /// The blob was deleted by this attempt. Reported with . + /// + BlobDeleted = 1, + + /// + /// The blob was already absent. Deletion is idempotent, so this is a success. Reported with + /// . + /// + BlobAlreadyAbsent = 2, + + /// + /// The blob did not carry the payload store's ownership marker, so it was left untouched. This is an + /// expected outcome, not a defect: the token text merely matched the v2 grammar. The tombstone is still + /// resolved because the blob is not the store's to delete. Reported with + /// . + /// + BlobNotStoreOwned = 3, + + /// + /// Network failure, timeout, storage outage, throttling, or a 5xx response. Reported with + /// . + /// + TransientStorageFailure = 10, + + /// + /// The registered payload store does not implement deletion. Every payload would fail the same way, so the + /// work is kept recoverable until an operator registers a store that can delete. Reported with + /// . + /// + StoreCannotDelete = 11, + + /// + /// The token is well formed but points at a storage account this worker's credential cannot reach. + /// Recoverable after a configuration or credential change. Reported with + /// . + /// + StorageAccountUnreachable = 12, + + /// + /// The token uses a recognized-but-newer version prefix this worker does not understand. Recoverable after + /// an SDK upgrade, so it earns a long defer rather than quarantine. Reported with + /// . + /// + UnsupportedTokenVersion = 13, + + /// + /// Authorization failed in a way that may be transient or reconfigurable (401/403). Reported with + /// . + /// + StorageAuthorizationFailed = 14, + + /// + /// The token carries a known version prefix but its body does not parse. Because the SDK and backend + /// control both sides of the protocol, this indicates a producer, corruption, or compatibility bug. + /// Reported with . + /// + MalformedToken = 20, + + /// + /// Storage rejected a request generated from a well-formed token as permanently invalid (HTTP 400, for + /// example InvalidUri / InvalidResourceName). Retrying can never succeed. Reported with + /// . + /// + InvalidStorageRequest = 21, + + /// + /// A legacy v1 token reached the worker. This is an invariant violation, because the backend excludes v1 + /// at insertion time. It cannot be safely deleted (no storage account in the token) and cannot be fixed by + /// retrying, so the evidence is preserved instead. Reported with + /// . + /// + LegacyV1Token = 22, +} diff --git a/src/Client/Core/LargePayloadPurgeResult.cs b/src/Client/Core/LargePayloadPurgeResult.cs new file mode 100644 index 00000000..f63f45fb --- /dev/null +++ b/src/Client/Core/LargePayloadPurgeResult.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable outcome of exactly one attempted large-payload blob deletion. Mirrors the +/// LargePayloadPurgeResult protobuf message but is safe to pass through the orchestration/activity +/// boundary. The backend owns retry scheduling: it deletes rows reported as +/// , reschedules +/// with a reason-appropriate next attempt, and moves +/// rows out of the active fetch. +/// +/// The backend partition that owns the tombstoned row. +/// The orchestration instance key the payload belonged to. +/// The backend identifier of the tombstoned payload row. +/// +/// The revision echoed unmodified from the fetched ; used by the backend +/// as a compare-and-swap guard. +/// +/// The disposition of the deletion attempt. +/// The stable reason code explaining the disposition. +/// +/// An optional bounded, sanitized storage status or error code for diagnostics (for example +/// BlobNotFound or 409). Never contains a token or raw exception text. +/// +public sealed record LargePayloadPurgeResult( + int PartitionId, + long InstanceKey, + long PayloadId, + long Revision, + LargePayloadPurgeDisposition Disposition, + LargePayloadPurgeReason Reason, + string? StorageErrorCode = null); diff --git a/src/Client/Core/LargePayloadTombstone.cs b/src/Client/Core/LargePayloadTombstone.cs new file mode 100644 index 00000000..c11a40db --- /dev/null +++ b/src/Client/Core/LargePayloadTombstone.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable representation of a tombstoned large-payload row whose external blob a credentialed caller +/// must delete. Mirrors the LargePayloadTombstone protobuf message but is safe to pass through the +/// orchestration/activity boundary. +/// +/// The backend partition that owns the tombstoned row. +/// The orchestration instance key the payload belonged to. +/// The backend identifier of the tombstoned payload row. +/// +/// The self-describing blob:v2:{fullBlobUrl} payload token whose backing blob should be deleted. +/// +/// +/// An optimistic-concurrency guard echoed back unmodified in the corresponding +/// so the backend can reject duplicate or stale reports without taking +/// a per-row lease. +/// +public sealed record LargePayloadTombstone( + int PartitionId, long InstanceKey, long PayloadId, string Token, long Revision); diff --git a/src/Client/Core/PayloadPurgeAck.cs b/src/Client/Core/PayloadPurgeAck.cs deleted file mode 100644 index 6e7788aa..00000000 --- a/src/Client/Core/PayloadPurgeAck.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.Client; - -/// -/// Serializable acknowledgement that the worker has deleted the blob for a tombstoned payload, so the -/// backend can hard-delete the soft-deleted row. Mirrors the PayloadPurgeAck protobuf message. -/// -/// The backend partition that owns the payload row. -/// The orchestration instance key the payload belongs to. -/// The backend identifier of the soft-deleted payload row. -public sealed record PayloadPurgeAck(int PartitionId, long InstanceKey, long PayloadId); diff --git a/src/Client/Core/TombstonedPayload.cs b/src/Client/Core/TombstonedPayload.cs deleted file mode 100644 index bb3f6896..00000000 --- a/src/Client/Core/TombstonedPayload.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.Client; - -/// -/// Serializable representation of a tombstoned payload the backend has soft-deleted and whose blob the -/// worker should delete. Mirrors the TombstonedPayload protobuf message but is safe to pass through -/// the orchestration/activity boundary. -/// -/// The backend partition that owns the payload row. -/// The orchestration instance key the payload belongs to. -/// The backend identifier of the soft-deleted payload row. -/// The externalized payload token whose backing blob should be deleted. -public sealed record TombstonedPayload(int PartitionId, long InstanceKey, long PayloadId, string Token); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 6e79b6b2..adaa2116 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -625,7 +625,7 @@ public override async Task> GetOrchestrationHistoryAsync( } /// - public override async Task> GetTombstonedPayloadsAsync( + public override async Task> GetLargePayloadTombstonesAsync( int limit, CancellationToken cancellation = default) { if (limit <= 0 || limit > 1000) @@ -634,59 +634,70 @@ public override async Task> GetTombstonedPayloadsAsync( nameof(limit), limit, "Limit must be greater than 0 and less than or equal to 1000."); } - P.GetTombstonedPayloadsResponse response; + P.GetLargePayloadTombstonesResponse response; try { - response = await this.sidecarClient.GetTombstonedPayloadsAsync( - new P.GetTombstonedPayloadsRequest { Limit = limit }, + response = await this.sidecarClient.GetLargePayloadTombstonesAsync( + new P.GetLargePayloadTombstonesRequest { Limit = limit }, cancellationToken: cancellation); } catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) { throw new OperationCanceledException( - $"The {nameof(this.GetTombstonedPayloadsAsync)} operation was canceled.", e, cancellation); + $"The {nameof(this.GetLargePayloadTombstonesAsync)} operation was canceled.", e, cancellation); } - List result = new(response.Payloads.Count); - foreach (P.TombstonedPayload payload in response.Payloads) + List result = new(response.Tombstones.Count); + foreach (P.LargePayloadTombstone tombstone in response.Tombstones) { - result.Add(new TombstonedPayload( - payload.PartitionId, payload.InstanceKey, payload.PayloadId, payload.Token)); + result.Add(new LargePayloadTombstone( + tombstone.PartitionId, + tombstone.InstanceKey, + tombstone.PayloadId, + tombstone.Token, + tombstone.Revision)); } return result; } /// - public override async Task AckPurgedPayloadsAsync( - IEnumerable acks, CancellationToken cancellation = default) + public override async Task ReportLargePayloadPurgeResultsAsync( + IEnumerable results, CancellationToken cancellation = default) { - Check.NotNull(acks); + Check.NotNull(results); - P.AckPurgedPayloadsRequest request = new(); - foreach (PayloadPurgeAck ack in acks) + P.ReportLargePayloadPurgeResultsRequest request = new(); + foreach (LargePayloadPurgeResult result in results) { - request.Acks.Add(new P.PayloadPurgeAck + request.Results.Add(new P.LargePayloadPurgeResult { - PartitionId = ack.PartitionId, - InstanceKey = ack.InstanceKey, - PayloadId = ack.PayloadId, + PartitionId = result.PartitionId, + InstanceKey = result.InstanceKey, + PayloadId = result.PayloadId, + Revision = result.Revision, + + // The managed enums declare the same numeric values as their protobuf counterparts, so the + // disposition and reason map across by value. + Disposition = (P.LargePayloadPurgeDisposition)result.Disposition, + Reason = (P.LargePayloadPurgeReason)result.Reason, + StorageErrorCode = result.StorageErrorCode ?? string.Empty, }); } - if (request.Acks.Count == 0) + if (request.Results.Count == 0) { return; } try { - await this.sidecarClient.AckPurgedPayloadsAsync(request, cancellationToken: cancellation); + await this.sidecarClient.ReportLargePayloadPurgeResultsAsync(request, cancellationToken: cancellation); } catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) { throw new OperationCanceledException( - $"The {nameof(this.AckPurgedPayloadsAsync)} operation was canceled.", e, cancellation); + $"The {nameof(this.ReportLargePayloadPurgeResultsAsync)} operation was canceled.", e, cancellation); } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs deleted file mode 100644 index 7ef6dbc8..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Microsoft.DurableTask.Client; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// Activity that acknowledges to the backend the payloads whose blobs the worker has deleted, so the backend -/// can hard-delete the soft-deleted rows. -/// -/// The Durable Task client used to acknowledge purged payloads to the backend. -/// The logger instance. -[DurableTask] -public class AckPurgedPayloadsActivity( - DurableTaskClient client, - ILogger logger) - : TaskActivity, object?> -{ - readonly DurableTaskClient client = Check.NotNull(client); - readonly ILogger logger = Check.NotNull(logger); - - /// - public override async Task RunAsync(TaskActivityContext context, List input) - { - if (input is null || input.Count == 0) - { - return null; - } - - await this.client.AckPurgedPayloadsAsync(input, CancellationToken.None); - this.logger.BlobPurgeAckedPayloads(input.Count); - return null; - } -} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index e90c5543..e811f46f 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -1,53 +1,45 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Globalization; +using System.Net; using Azure; +using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Activity that deletes a single externalized payload blob given its token. Deletion is idempotent, so -/// re-delivered tokens and concurrent workers are safe. +/// Activity that deletes a single externalized payload blob given its token, and classifies the attempt as +/// , , or +/// with a stable reason code. Deletion is idempotent, +/// so re-delivered tokens and concurrent workers are safe. /// /// -/// Outcome classification, verified against the Azure.Storage.Blobs / Azure.Core exception model (not -/// assumed): +/// The split between retry and quarantine is whether the failure can self-heal, verified against the +/// Azure.Storage.Blobs / Azure.Core exception model (not assumed): /// /// /// The Azure SDK already retries transient failures internally (connection errors plus HTTP /// 408/429/500/502/503/504, with exponential backoff), so any exception that escapes -/// means those built-in retries were already exhausted. +/// means those built-in retries were already exhausted. It is still +/// classified as retryable, because the backend - not this activity - owns retry scheduling and can defer the +/// row past a storage outage. /// /// -/// Legacy blob:v1: tokens are discarded without a delete attempt: a v1 token carries only a container -/// name, not the storage account, so a delete against the currently-configured account cannot be verified - if -/// the store has been repointed the delete would report success while the real blob survives elsewhere. The -/// backend ack protocol has no "skip" status (an un-acked row is re-served every cycle), so the token is acked -/// to keep the pipeline moving and logged at error level as the operator's recovery pointer. A current backend -/// hard-deletes v1 rows instead of tombstoning them, so this branch is a defensive guard for an older backend -/// build or a row tombstoned before that fix, not the expected path. +/// Quarantine is reserved for deterministic failures and protocol violations that retrying can never fix: a +/// known version prefix whose body does not parse, a request storage rejected as permanently invalid, and a +/// legacy v1 token. Quarantine preserves the row and its token as durable evidence, so a permanent failure +/// neither blocks the queue nor destroys the only record of the blob. /// /// -/// Permanent failures are discarded (acked so the backend clears the row) because retrying can never succeed: -/// an from the store's token decode - a genuinely malformed or unrecognized -/// token (v1 tokens are gated out above and never reach the store); a -/// with 400 (for example -/// InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules); and a -/// when a v2 token points at a storage account the configured credential -/// cannot reach (connection-string / account-key auth is account-specific). The backend batch is cursor-less, -/// so an undroppable row would otherwise re-stream every cycle and block the pipeline head-of-line; the -/// account-unreachable case is logged at error level so an operator can reconcile it. -/// -/// -/// Everything else is treated as transient and leaves the payload tombstoned to retry on a later cycle: -/// throttling / 5xx that outlived the SDK's retries, 403 authorization failures (which need an operator -/// credential fix rather than dropping data), timeouts / cancellation, and a -/// from a misconfigured store that cannot delete at all (retried, not dropped, so the work is recoverable once -/// a deleting store is registered). A blob is never dropped on an uncertain error, and a single bad token never -/// fails the whole batch. +/// A blob is never deleted on an uncertain error, and a single bad token never fails the whole batch: a +/// failure is returned as a disposition rather than thrown. /// /// +/// Per design §7 no log here carries the token or raw exception text - a token exposes the storage account, +/// container, and blob path. Diagnostics are the stable reason enum plus a bounded, sanitized storage error +/// code; the token itself is preserved on the backend's quarantined row. /// /// The payload store used to delete blobs. /// The logger instance. @@ -55,78 +47,145 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class DeleteExternalBlobActivity( PayloadStore store, ILogger logger) - : TaskActivity + : TaskActivity { readonly PayloadStore store = Check.NotNull(store); readonly ILogger logger = Check.NotNull(logger); /// - public override async Task RunAsync(TaskActivityContext context, string input) + public override async Task RunAsync(TaskActivityContext context, string input) { Check.NotNullOrEmpty(input, nameof(input)); - if (input.StartsWith(BlobPayloadStore.TokenPrefixV1, StringComparison.Ordinal)) + BlobPurgeOutcome outcome = await this.DeleteAsync(input); + + switch (outcome.Disposition) + { + case LargePayloadPurgeDisposition.Quarantined: + this.logger.BlobPurgeDeleteQuarantined(outcome.Reason.ToString(), outcome.StorageErrorCode); + break; + case LargePayloadPurgeDisposition.Retry: + this.logger.BlobPurgeDeleteRetryable(outcome.Reason.ToString(), outcome.StorageErrorCode); + break; + case LargePayloadPurgeDisposition.Deleted + when outcome.Reason == LargePayloadPurgeReason.BlobNotStoreOwned: + this.logger.BlobPurgeBlobNotStoreOwned(); + break; + } + + return outcome; + } + + /// + /// Extracts a bounded, sanitized storage error code for diagnostics. The service's own error code (for + /// example BlobNotFound) is a fixed vocabulary and the numeric status is the fallback, so neither + /// can carry a token or raw exception text. + /// + static string SanitizeErrorCode(RequestFailedException exception) => + string.IsNullOrEmpty(exception.ErrorCode) + ? exception.Status.ToString(CultureInfo.InvariantCulture) + : exception.ErrorCode; + + async Task DeleteAsync(string token) + { + // Classify the token's version prefix before consulting the store. The store reports every token it + // cannot decode as the same ArgumentException, but the three cases have opposite dispositions, so they + // are separated here, where the prefix is still visible. + if (token.StartsWith(BlobPayloadStore.TokenPrefixV1, StringComparison.Ordinal)) { - // Auto-purge deliberately does not act on legacy v1 tokens. A v1 token carries only a container - // *name* - not the storage account - so a delete against the currently-configured account cannot be - // verified: if the store has since been repointed, DeleteIfExistsAsync returns false and the purge - // would silently report success while the real blob survives in the old account. Rather than delete - // on an unverifiable pointer, the token is discarded. The backend ack protocol carries no "skip" - // status (PayloadPurgeAck is just partition/instance/payload id, and an un-acked row is re-served by - // an uncursored TOP(N) query every cycle), so declining without acking would permanently block the - // pipeline. The full token is logged at error level so it remains a recoverable pointer. A current - // backend hard-deletes v1 rows instead of tombstoning them, so reaching this branch means an older - // backend build or a row tombstoned before that fix - it is a defensive guard, not the expected path. - this.logger.BlobPurgeDeleteV1TokenUnsupported(input); - return BlobDeleteResult.Discarded; + // A v1 token carries a container *name* but not the storage account, so a delete against the + // currently-configured account cannot be verified: if the store has since been repointed, + // DeleteIfExists returns false and the purge would falsely report success while the real blob + // survives in the old account. Retrying cannot fix that, and a success-shaped discard would destroy + // the only durable record of the blob, so the row is quarantined instead - the backend preserves + // its token as evidence and stops polling it. The backend excludes v1 at insertion time, so + // reaching this branch is an invariant violation rather than an expected path. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.LegacyV1Token); + } + + if (!token.StartsWith(BlobPayloadStore.TokenPrefixV2, StringComparison.Ordinal)) + { + // An unrecognized prefix is most likely a token written by a newer SDK than this worker runs. That + // recovers after an upgrade, so it earns a deferral rather than quarantine. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.UnsupportedTokenVersion); } try { - await this.store.DeleteAsync(input, CancellationToken.None); - return BlobDeleteResult.Deleted; + PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, CancellationToken.None); + return outcome switch + { + PayloadDeleteOutcome.Deleted => new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobDeleted), + PayloadDeleteOutcome.AlreadyAbsent => new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobAlreadyAbsent), + + // The blob exists but this store never wrote it, so it was left untouched. That is an expected + // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it + // would fill the quarantine set with non-defects. The tombstone is still resolved, because a + // blob the store does not own is not the store's to delete. + _ => new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobNotStoreOwned), + }; + } + catch (ArgumentException) + { + // The prefix gate above proves this is a v2 token, so the only remaining decode failure is a v2 + // body that does not parse. The SDK and backend control both sides of the protocol, so that + // indicates a producer, corruption, or compatibility bug; retrying can never fix it. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.MalformedToken); + } + catch (NotSupportedException) + { + // The registered store does not implement deletion. Every payload would fail the same way, so the + // work is kept recoverable until an operator registers a store that can delete. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StoreCannotDelete); } - catch (ArgumentException ex) + catch (PayloadStorageException) { - // The token is malformed or points at a different container; it can never succeed. Discard it so - // the backend clears the row instead of re-streaming the same poison token every cycle. - this.logger.BlobPurgeDeleteDiscarded(ex, input); - return BlobDeleteResult.Discarded; + // The token is well formed but points at a storage account this worker's credential cannot reach + // (account-key auth is account-specific). Recoverable after a configuration or credential change, + // so it is deferred rather than discarded. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageAccountUnreachable); } - catch (RequestFailedException ex) when (ex.Status == 400) + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.BadRequest) { - // Service rejected the request as permanently invalid (e.g. InvalidUri / InvalidResourceName - the - // decoded blob name violates Azure naming rules). Retrying can never succeed, so discard it like a - // poison token: ack so the backend clears the row instead of re-streaming it forever. - this.logger.BlobPurgeDeleteDiscarded(ex, input); - return BlobDeleteResult.Discarded; + // Storage rejected a request generated from a well-formed token as permanently invalid (for + // example InvalidUri / InvalidResourceName). Retrying can never succeed. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Quarantined, + LargePayloadPurgeReason.InvalidStorageRequest, + SanitizeErrorCode(ex)); } - catch (NotSupportedException ex) + catch (RequestFailedException ex) when ( + ex.Status == (int)HttpStatusCode.Unauthorized || ex.Status == (int)HttpStatusCode.Forbidden) { - // The registered store does not implement deletion (PayloadStore.DeleteAsync is virtual and its base - // implementation throws). This is a misconfiguration, not poison data: every payload would fail the - // same way, so acking would hard-delete the backend's entire record of what still needs cleanup while - // every blob survives. Keep the payload tombstoned so the work is recoverable once an operator - // registers a store that can delete. - this.logger.BlobPurgeDeleteNotSupported(ex, input); - return BlobDeleteResult.Retry; + // Authorization can be transient or fixed by reconfiguration, so it stays recoverable rather than + // dropping data an operator can still reclaim. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, + LargePayloadPurgeReason.StorageAuthorizationFailed, + SanitizeErrorCode(ex)); } - catch (PayloadStorageException ex) + catch (RequestFailedException ex) { - // The token is well-formed but points at a storage account this worker's credential cannot reach - // (cross-account without AAD). Retrying can never succeed from this process, and because the backend - // streams tombstones with an uncursored TOP(N) query, leaving it un-acked would re-serve the same - // token every cycle and permanently block the purge pipeline. Discard it so the row is cleared, and - // log at Error so an operator can reclaim the orphaned blob out-of-band. - this.logger.BlobPurgeDeleteUnreachable(ex, input); - return BlobDeleteResult.Discarded; + // Throttling, 5xx, and anything else the service reported, including a failed If-Match on the + // ownership check: transient by default. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, + LargePayloadPurgeReason.TransientStorageFailure, + SanitizeErrorCode(ex)); } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { - // Transient failure: leave the payload tombstoned so a later purge cycle can retry it. A single - // bad token must not fail the whole batch. - this.logger.BlobPurgeDeleteFailed(ex, input); - return BlobDeleteResult.Retry; + // Timeouts, cancellation, and network failures. A blob is never dropped on an uncertain error. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.TransientStorageFailure); } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs new file mode 100644 index 00000000..273989ea --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that fetches a bounded batch of due large-payload tombstones from the backend for the auto-purge +/// job to delete. +/// +/// The Durable Task client used to query the backend for tombstones. +/// The logger instance. +[DurableTask] +public class GetLargePayloadTombstonesActivity( + DurableTaskClient client, + ILogger logger) + : TaskActivity> +{ + readonly DurableTaskClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task> RunAsync(TaskActivityContext context, int input) + { + int limit = input; + List tombstones = + await this.client.GetLargePayloadTombstonesAsync(limit, CancellationToken.None); + this.logger.BlobPurgeFetchedTombstones(tombstones.Count); + return tombstones; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs deleted file mode 100644 index 838b293a..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Microsoft.DurableTask.Client; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// Activity that fetches a batch of tombstoned payloads from the backend for the auto-purge job to delete. -/// -/// The Durable Task client used to query the backend for tombstoned payloads. -/// The logger instance. -[DurableTask] -public class GetTombstonedPayloadsActivity( - DurableTaskClient client, - ILogger logger) - : TaskActivity> -{ - readonly DurableTaskClient client = Check.NotNull(client); - readonly ILogger logger = Check.NotNull(logger); - - /// - public override async Task> RunAsync(TaskActivityContext context, int input) - { - int limit = input; - List payloads = - await this.client.GetTombstonedPayloadsAsync(limit, CancellationToken.None); - this.logger.BlobPurgeFetchedTombstones(payloads.Count); - return payloads; - } -} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs new file mode 100644 index 00000000..7003b7c9 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that reports the outcome of every attempted blob deletion to the backend, so it can delete the +/// resolved tombstones, reschedule the retryable ones, and quarantine the rest. Every attempted row is +/// reported, not only the successful ones: the backend owns retry scheduling, so a row it hears nothing about +/// would simply be re-served unchanged on the next cycle. +/// +/// The Durable Task client used to report purge results to the backend. +/// The logger instance. +[DurableTask] +public class ReportLargePayloadPurgeResultsActivity( + DurableTaskClient client, + ILogger logger) + : TaskActivity, object?> +{ + readonly DurableTaskClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task RunAsync( + TaskActivityContext context, List input) + { + if (input is null || input.Count == 0) + { + return null; + } + + await this.client.ReportLargePayloadPurgeResultsAsync(input, CancellationToken.None); + this.logger.BlobPurgeReportedResults(input.Count); + return null; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index ca4cce32..0138da35 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -22,7 +22,7 @@ static class BlobPurgeConstants /// /// The maximum batch size the auto-purge job may request per cycle. Mirrors the gRPC - /// GetTombstonedPayloadsAsync contract, which rejects limits greater than 1000. + /// GetLargePayloadTombstones contract, which rejects limits greater than 1000. /// public const int MaxBatchSize = 1000; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index f2989b9f..c20cc5d4 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -19,14 +19,14 @@ static partial class Logs [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] public static partial void BlobPurgeJobOrchestratorStopping(this ILogger logger, string? jobId, string status); - [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Failed to delete externalized payload blob for token '{token}'; leaving it tombstoned for a later purge cycle.")] - public static partial void BlobPurgeDeleteFailed(this ILogger logger, Exception exception, string token); + [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Blob payload auto-purge quarantined a payload; reason '{reason}', storage error code '{storageErrorCode}'. The failure is deterministic and cannot succeed on a retry. The backend preserves the tombstone row and its token as evidence and stops polling it.")] + public static partial void BlobPurgeDeleteQuarantined(this ILogger logger, string reason, string? storageErrorCode); [LoggerMessage(EventId = 814, Level = LogLevel.Debug, Message = "Blob payload auto-purge fetched {count} tombstoned payload(s) from the backend.")] public static partial void BlobPurgeFetchedTombstones(this ILogger logger, int count); - [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge acknowledged {count} purged payload(s) to the backend.")] - public static partial void BlobPurgeAckedPayloads(this ILogger logger, int count); + [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge reported {count} purge result(s) to the backend.")] + public static partial void BlobPurgeReportedResults(this ILogger logger, int count); [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] public static partial void BlobPurgeJobEnsured(this ILogger logger); @@ -34,21 +34,15 @@ static partial class Logs [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); - [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Discarding poison externalized payload token '{token}'; it can never be deleted, acknowledging it so the backend can clear the row.")] - public static partial void BlobPurgeDeleteDiscarded(this ILogger logger, Exception exception, string token); + [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; reason '{reason}', storage error code '{storageErrorCode}'. The backend reschedules the tombstone for a later attempt.")] + public static partial void BlobPurgeDeleteRetryable(this ILogger logger, string reason, string? storageErrorCode); [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); - [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' points at a storage account the configured credential cannot reach; the blob cannot be deleted by this worker and will be orphaned. Acknowledging it so the backend can clear the row - reclaim the blob manually or reconfigure the payload store with identity (AAD) authentication that can access both accounts.")] - public static partial void BlobPurgeDeleteUnreachable(this ILogger logger, Exception exception, string token); - - [LoggerMessage(EventId = 822, Level = LogLevel.Error, Message = "Received a legacy v1 externalized payload token '{token}' from the backend, which is unexpected: a current backend hard-deletes v1 payload rows instead of tombstoning them, so this indicates either an older backend build or a row that was tombstoned before that fix. Auto-purge does not delete v1 tokens because a v1 token identifies the container by name only and not the storage account, so the delete cannot be verified; the backing blob is NOT deleted and the row is acknowledged so the purge pipeline is not blocked. Reclaim the blob using the container and blob name in the token above, and upgrade to an SDK version that writes self-describing v2 tokens.")] - public static partial void BlobPurgeDeleteV1TokenUnsupported(this ILogger logger, string token); + [LoggerMessage(EventId = 821, Level = LogLevel.Warning, Message = "An externalized payload blob does not carry this store's ownership marker, so it was left untouched; the tombstone is still resolved. This is expected for payloads written before the marker shipped, and for blobs the store never created whose token text matches the payload token grammar.")] + public static partial void BlobPurgeBlobNotStoreOwned(this ILogger logger); [LoggerMessage(EventId = 823, Level = LogLevel.Error, Message = "Blob payload auto-purge is enabled but the registered PayloadStore ('{storeType}') is not an Azure Blob payload store and cannot delete payloads. The auto-purge job was not started; externalized payloads will not be reclaimed. Register the Azure Blob payload store, or disable AutoPurge.")] public static partial void BlobPurgeStoreCannotDelete(this ILogger logger, string? storeType); - - [LoggerMessage(EventId = 824, Level = LogLevel.Error, Message = "The registered PayloadStore does not support deleting payloads, so externalized payload token '{token}' cannot be purged. Leaving it tombstoned; register an Azure Blob payload store on the worker or disable AutoPurge.")] - public static partial void BlobPurgeDeleteNotSupported(this ILogger logger, Exception exception, string token); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs deleted file mode 100644 index d3386be5..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// The outcome of attempting to delete a single externalized payload blob during an auto-purge cycle. -/// -public enum BlobDeleteResult -{ - /// - /// The blob was deleted, or was already gone. The payload should be acknowledged so the backend can - /// hard-delete the row. - /// - Deleted, - - /// - /// The token is permanently invalid (poison) and can never be deleted. The payload should still be - /// acknowledged so the backend clears the stuck row instead of re-streaming the same token forever. - /// - Discarded, - - /// - /// A transient failure occurred. The payload is left tombstoned so a later purge cycle can retry it. - /// - Retry, -} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs new file mode 100644 index 00000000..e5c563ec --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// The outcome of attempting to delete a single externalized payload blob during an auto-purge cycle. The +/// orchestrator combines it with the tombstone's identity and revision to build the reported +/// . +/// +/// Whether the row is resolved, should be retried, or must be quarantined. +/// The stable reason code explaining the disposition. +/// +/// An optional bounded, sanitized storage status or error code for diagnostics. Never a token or raw +/// exception text. +/// +public sealed record BlobPurgeOutcome( + LargePayloadPurgeDisposition Disposition, + LargePayloadPurgeReason Reason, + string? StorageErrorCode = null); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 50758e60..7d1730c6 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -17,9 +17,10 @@ public sealed record BlobPurgeJobRunRequest( EntityInstanceId JobEntityId, int PurgeBatchSize, int ProcessedCycles = 0); /// -/// Perpetual orchestrator that drains tombstoned payloads from the backend, deletes their blobs with capped -/// parallelism, and acknowledges the successful deletions so the backend can hard-delete the rows. It idles -/// on a timer when there is nothing to purge and continues-as-new periodically to keep its history small. +/// Perpetual orchestrator that drains due large-payload tombstones from the backend, deletes their blobs with +/// capped parallelism, and reports every outcome so the backend can resolve, reschedule, or quarantine each +/// row. It idles on a timer when there is nothing to purge and continues-as-new periodically to keep its +/// history small. /// [DurableTask] public class BlobPurgeJobOrchestrator : TaskOrchestrator @@ -66,8 +67,8 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator tombstones = await context.CallActivityAsync>( - nameof(GetTombstonedPayloadsActivity), + List tombstones = await context.CallActivityAsync>( + nameof(GetLargePayloadTombstonesActivity), batchSize, new TaskOptions(PurgeActivityRetryPolicy)); @@ -78,26 +79,34 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator acks = await this.DeleteBatchAsync(context, tombstones); + List results = await this.DeleteBatchAsync(context, tombstones); - if (acks.Count > 0) - { - await context.CallActivityAsync( - nameof(AckPurgedPayloadsActivity), - acks, - new TaskOptions(PurgeActivityRetryPolicy)); + // Every attempted row produces a result, including the retryable ones: the backend owns retry + // scheduling, so it needs to hear about a failure to defer the row. Reporting unconditionally + // is what keeps a failing row from being re-served unchanged on the very next cycle. + await context.CallActivityAsync( + nameof(ReportLargePayloadPurgeResultsActivity), + results, + new TaskOptions(PurgeActivityRetryPolicy)); + + // Two different questions, deliberately not conflated. Progress counts only payloads that were + // actually purged; the backoff decision asks whether ANY row left the retry queue, because a + // quarantined row also stops being re-served even though nothing was reclaimed. + int purged = CountDisposition(results, LargePayloadPurgeDisposition.Deleted); + int resolved = results.Count - CountDisposition(results, LargePayloadPurgeDisposition.Retry); + if (purged > 0) + { await context.Entities.CallEntityAsync( - input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)acks.Count); + input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)purged); } - else + + if (resolved == 0) { - // Nothing in this batch could be acknowledged: every delete returned Retry (e.g. a storage - // outage or throttling). Deletes report failure as a return value rather than an exception, - // so no retry policy or backoff applies on that path. The backend serves tombstones with an - // uncursored TOP(N) query, so continuing immediately would refetch the identical rows and - // re-attempt the identical deletes in a tight loop for as long as the outage lasts. Back off - // before trying again. + // The whole batch came back retryable (e.g. a storage outage or throttling). Deletes report + // failure as a return value rather than an exception, so no activity retry policy applies on + // that path. Continuing immediately would refetch and re-attempt in a tight loop for as long + // as the outage lasts, so back off before the next cycle. await context.CreateTimer(ErrorBackoff, default); } } @@ -112,56 +121,71 @@ await context.Entities.CallEntityAsync( } } - async Task> DeleteBatchAsync( - TaskOrchestrationContext context, List tombstones) + static int CountDisposition( + List results, LargePayloadPurgeDisposition disposition) + { + int count = 0; + foreach (LargePayloadPurgeResult result in results) + { + if (result.Disposition == disposition) + { + count++; + } + } + + return count; + } + + static async Task DrainAsync( + List> tasks, List results) + { + LargePayloadPurgeResult[] completed = await Task.WhenAll(tasks); + results.AddRange(completed); + } + + async Task> DeleteBatchAsync( + TaskOrchestrationContext context, List tombstones) { - List acks = new(tombstones.Count); - List> tasks = new(); + List results = new(tombstones.Count); + List> tasks = new(); - foreach (TombstonedPayload tombstone in tombstones) + foreach (LargePayloadTombstone tombstone in tombstones) { tasks.Add(this.DeleteOneAsync(context, tombstone)); if (tasks.Count >= MaxParallelDeletes) { - await DrainAsync(tasks, acks); + await DrainAsync(tasks, results); tasks.Clear(); } } if (tasks.Count > 0) { - await DrainAsync(tasks, acks); + await DrainAsync(tasks, results); } - return acks; - } - - static async Task DrainAsync(List> tasks, List acks) - { - DeleteOutcome[] outcomes = await Task.WhenAll(tasks); - foreach (DeleteOutcome outcome in outcomes) - { - // Acknowledge blobs that were deleted (or already gone) and poison tokens that can never succeed - // so the backend can hard-delete their rows; transient failures stay tombstoned to retry. - if (outcome.ShouldAck) - { - acks.Add(outcome.Ack); - } - } + return results; } - async Task DeleteOneAsync(TaskOrchestrationContext context, TombstonedPayload tombstone) + async Task DeleteOneAsync( + TaskOrchestrationContext context, LargePayloadTombstone tombstone) { - BlobDeleteResult result = await context.CallActivityAsync( + BlobPurgeOutcome outcome = await context.CallActivityAsync( nameof(DeleteExternalBlobActivity), tombstone.Token, new TaskOptions(PurgeActivityRetryPolicy)); - return new DeleteOutcome( - result != BlobDeleteResult.Retry, - new PayloadPurgeAck(tombstone.PartitionId, tombstone.InstanceKey, tombstone.PayloadId)); + // The revision is echoed back unchanged so the backend can detect a tombstone that was rewritten while + // this attempt was in flight and ignore the stale result. Retry scheduling is the backend's job, so no + // next-attempt time is computed here. + return new LargePayloadPurgeResult( + tombstone.PartitionId, + tombstone.InstanceKey, + tombstone.PayloadId, + tombstone.Revision, + outcome.Disposition, + outcome.Reason, + outcome.StorageErrorCode); } - - readonly record struct DeleteOutcome(bool ShouldAck, PayloadPurgeAck Ack); } diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index c63db137..142f96eb 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -84,19 +84,24 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB } opt.Capabilities.Add(P.WorkerCapability.LargePayloads); + + // Item 2 of the pre-merge checklist: the resolved AutoPurge value rides the work-item + // handshake so the backend only tombstones payloads for a task hub whose workers opted in. + // Absent means "no opinion"; an explicit true/false is the customer's choice. + opt.LargePayloadAutoPurgeEnabled = opts.AutoPurge; }); // Register the entity/orchestrators/activities that run the singleton auto-purge job. These are // ALWAYS registered (not gated on AutoPurge) so that a client-enabled job always has something to - // execute here. The purge activities fetch/ack via the injected DurableTaskClient. + // execute here. The purge activities fetch/report via the injected DurableTaskClient. builder.AddTasks(r => { r.AddEntity(); r.AddOrchestrator(); r.AddOrchestrator(); - r.AddActivity(); + r.AddActivity(); r.AddActivity(); - r.AddActivity(); + r.AddActivity(); }); return builder; diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index d67f967c..9a22cd03 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -28,7 +28,28 @@ public sealed class BlobPayloadStore : PayloadStore /// account. Auto-purge uses this to detect and skip v1 tokens. /// internal const string TokenPrefixV1 = "blob:v1:"; - const string TokenPrefixV2 = "blob:v2:"; + + /// + /// The prefix of self-describing v2 payload tokens, which carry the blob's absolute URI including the + /// storage account. Auto-purge uses this to tell a malformed v2 token (a protocol defect) apart from an + /// unrecognized version prefix (a token written by a newer SDK). + /// + internal const string TokenPrefixV2 = "blob:v2:"; + + /// + /// The metadata name of the ownership marker written on every blob this store creates. Recognizing a + /// token proves only that its text matches the store's grammar; the marker is what proves the store + /// actually wrote the blob, so a customer's own blob is never deleted just because an orchestration + /// referenced it. Azure requires metadata names to follow the naming rules for C# identifiers, so the + /// marker is spelled with an underscore rather than a hyphen. + /// + internal const string OwnershipMarkerName = "managed_by"; + + /// + /// The fixed value of the ownership marker written on every blob this store creates. + /// + internal const string OwnershipMarkerValue = "dts"; + const string ContentEncodingGzip = "gzip"; const int MaxRetryAttempts = 8; const int BaseDelayMs = 250; @@ -124,6 +145,7 @@ public override async Task UploadAsync(string payLoad, CancellationToken BlobOpenWriteOptions writeOptions = new() { HttpHeaders = new BlobHttpHeaders { ContentEncoding = ContentEncodingGzip }, + Metadata = CreateOwnershipMetadata(), }; using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, cancellationToken); using GZipStream compressedBlobStream = new(blobStream, System.IO.Compression.CompressionLevel.Optimal, leaveOpen: true); @@ -137,7 +159,15 @@ public override async Task UploadAsync(string payLoad, CancellationToken } else { - using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); + // The uncompressed path still needs write options purely to carry the ownership marker: + // the marker must be written by every path that creates a blob, or auto-purge would later + // decline to delete the store's own uncompressed payloads. It rides along in the PUT the + // upload already issues, so it costs no extra request. + BlobOpenWriteOptions writeOptions = new() + { + Metadata = CreateOwnershipMetadata(), + }; + using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, cancellationToken); // using MemoryStream payloadStream = new(payloadBuffer, writable: false); // await payloadStream.CopyToAsync(blobStream, bufferSize: DefaultCopyBufferSize, cancellationToken); @@ -203,7 +233,7 @@ public override async Task DownloadAsync(string token, CancellationToken } /// - public override async Task DeleteAsync(string token, CancellationToken cancellationToken) + public override async Task DeleteAsync(string token, CancellationToken cancellationToken) { DecodeTokenResult decoded = DecodeToken(token); @@ -238,12 +268,43 @@ public override async Task DeleteAsync(string token, CancellationToken cancellat "account-key credentials are account-specific and cannot delete in another account."); } + // Recognizing the token proves only that its text matches this store's grammar - not that this store + // wrote the blob. A customer may keep an expensive dataset in Blob Storage and have orchestrations + // reference it by URL; deleting that would destroy data the store never created. So ownership is read + // from the object itself before anything is deleted. + BlobProperties properties; + try + { + Response response = await blob.GetPropertiesAsync( + conditions: null, cancellationToken: cancellationToken); + properties = response.Value; + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) + { + // Already gone. Deletion is idempotent, so a re-delivered tombstone or a concurrent worker + // replica that won the race is a success, not an error. + return PayloadDeleteOutcome.AlreadyAbsent; + } + + if (!HasOwnershipMarker(properties.Metadata)) + { + // Positive evidence that the blob is customer-owned: leave it untouched. The caller still resolves + // the payload reference, because a blob this store never wrote is not this store's to delete. + return PayloadDeleteOutcome.NotStoreOwned; + } + + // Pair the ownership read with the delete using the ETag from that same read. If anything rewrites the + // blob in between - including a customer overwriting it with content that no longer carries the marker + // - the If-Match condition fails the delete instead of removing the newer content, so the read-then- + // delete behaves as a single check-and-delete without taking a lease. // Idempotent by design: DeleteIfExistsAsync returns false (rather than throwing) when the blob is // already gone, so re-delivered tombstones and concurrent purges from multiple worker replicas are safe. - await blob.DeleteIfExistsAsync( + Response deleted = await blob.DeleteIfExistsAsync( DeleteSnapshotsOption.IncludeSnapshots, - conditions: null, + conditions: new BlobRequestConditions { IfMatch = properties.ETag }, cancellationToken: cancellationToken); + + return deleted.Value ? PayloadDeleteOutcome.Deleted : PayloadDeleteOutcome.AlreadyAbsent; } /// @@ -309,6 +370,38 @@ internal static DecodeTokenResult DecodeToken(string token) throw new ArgumentException("Invalid external payload token.", nameof(token)); } + /// + /// Creates the ownership metadata stamped on every blob this store writes, so a later purge can prove the + /// store created the blob before deleting it. + /// + static IDictionary CreateOwnershipMetadata() => + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [OwnershipMarkerName] = OwnershipMarkerValue, + }; + + /// + /// Returns whether the supplied blob metadata carries this store's ownership marker. Azure treats metadata + /// names as case-insensitive, so the lookup is too. + /// + static bool HasOwnershipMarker(IDictionary? metadata) + { + if (metadata is null) + { + return false; + } + + foreach (KeyValuePair entry in metadata) + { + if (string.Equals(entry.Key, OwnershipMarkerName, StringComparison.OrdinalIgnoreCase)) + { + return string.Equals(entry.Value, OwnershipMarkerValue, StringComparison.Ordinal); + } + } + + return false; + } + static async Task WritePayloadAsync(byte[] payloadBuffer, Stream target, CancellationToken cancellationToken) { #if NETSTANDARD2_0 diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs new file mode 100644 index 00000000..532cbcd9 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask; + +/// +/// The outcome of deleting a payload through . All three values are +/// successful terminal outcomes; failures are surfaced as exceptions instead. +/// +public enum PayloadDeleteOutcome +{ + /// + /// The payload's backing object was deleted by this call. + /// + Deleted, + + /// + /// The payload's backing object was already absent. Deletion is idempotent, so this is a success. + /// + AlreadyAbsent, + + /// + /// The backing object exists but does not carry the store's ownership marker, so the store did not + /// create it and left it untouched. The payload reference is still resolved, because an object the + /// store never wrote is not the store's to delete. + /// + NotStoreOwned, +} diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs index 0ae5ccb6..ee6f22b2 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs @@ -32,11 +32,17 @@ public abstract class PayloadStore /// The default implementation throws . Stores that externalize /// payloads to deletable storage (for example Azure Blob Storage) should override it. It is declared /// virtual rather than abstract so that adding it does not break existing external subclasses. + /// Implementations must delete only objects they created; an object that carries no proof of the + /// store's ownership must be left untouched and reported as + /// . /// /// The opaque reference token. /// Cancellation token. - /// A task that completes when the payload has been deleted (or was already absent). - public virtual Task DeleteAsync(string token, CancellationToken cancellationToken) => + /// + /// The outcome of the deletion: whether the object was deleted, was already absent, or was left in + /// place because the store does not own it. + /// + public virtual Task DeleteAsync(string token, CancellationToken cancellationToken) => throw new NotSupportedException( $"This {nameof(PayloadStore)} implementation does not support deleting payloads."); diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index af065941..fc007e9a 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -786,6 +786,18 @@ service TaskHubSidecarService { rpc PurgeInstances(PurgeInstancesRequest) returns (PurgeInstancesResponse); rpc GetWorkItems(GetWorkItemsRequest) returns (stream WorkItem); + + // Returns a bounded, deterministically ordered batch of due large-payload tombstones whose + // external blobs the worker must delete. Scoped to the caller's authenticated task hub. + // Only rows that are pending and whose next attempt time has arrived are returned; a row stays + // pending until its outcome is reported, so this is safe under retries and duplicate callers. + rpc GetLargePayloadTombstones(GetLargePayloadTombstonesRequest) returns (GetLargePayloadTombstonesResponse); + + // Reports the outcome of each attempted blob deletion. The backend owns retry scheduling: it + // deletes rows reported as DELETED, reschedules RETRY with a reason-appropriate next attempt, + // and moves QUARANTINED rows out of the active fetch while preserving their evidence. + rpc ReportLargePayloadPurgeResults(ReportLargePayloadPurgeResultsRequest) returns (ReportLargePayloadPurgeResultsResponse); + rpc CompleteActivityTask(ActivityResponse) returns (CompleteTaskResponse); rpc CompleteOrchestratorTask(OrchestratorResponse) returns (CompleteTaskResponse); rpc CompleteEntityTask(EntityBatchResult) returns (CompleteTaskResponse); @@ -823,53 +835,133 @@ service TaskHubSidecarService { // "Skip" graceful termination of orchestrations by immediately changing their status in storage to "terminated". // Note that a maximum of 500 orchestrations can be terminated at a time using this method. rpc SkipGracefulOrchestrationTerminations(SkipGracefulOrchestrationTerminationsRequest) returns (SkipGracefulOrchestrationTerminationsResponse); - - // Returns a batch of blob-externalized payload tombstones the backend has soft-deleted so the worker - // can delete the corresponding blobs from customer storage (the backend has no storage credentials of - // its own). The worker deletes each blob and then calls AckPurgedPayloads so the backend can - // hard-delete the soft-deleted rows. This is an opt-in, worker-driven pull model. - rpc GetTombstonedPayloads(GetTombstonedPayloadsRequest) returns (GetTombstonedPayloadsResponse); - - // Acknowledges that the worker has deleted the blobs for the identified payloads so the backend can - // hard-delete the soft-deleted rows. - rpc AckPurgedPayloads(AckPurgedPayloadsRequest) returns (AckPurgedPayloadsResponse); } -// Server -> client. Identifies a blob-externalized payload that the backend has soft-deleted and whose -// blob the worker should delete from customer storage. -message TombstonedPayload { +// server -> client: one tombstoned large-payload row whose external blob the worker must delete. +message LargePayloadTombstone { int32 partitionId = 1; int64 instanceKey = 2; int64 payloadId = 3; - // The externalized payload token (e.g. "blob:v1::") whose backing blob should be deleted. + + // A self-describing SDK v2 token: "blob:v2:{fullBlobUrl}". + // Legacy v1 tokens are never tombstoned: v1 carries a container name but not the storage + // account, so a delete against the configured account cannot be verified. The backend + // hard-deletes v1 payload rows instead (design §8). string token = 4; + + // Optimistic-concurrency guard. The worker echoes this value back unmodified in the result so + // the backend can reject duplicate or stale reports without taking a per-row lease. + int64 revision = 5; +} + +// The outcome of a single blob deletion attempt. The split is by whether a failure can self-heal. +enum LargePayloadPurgeDisposition { + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; + + // Terminal success. The blob was deleted, was already absent, or was deliberately left in + // place because it is not owned by the payload store (design §5.5). The backend deletes the + // tombstone in all three cases. + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; + + // The failure may self-heal, so the row stays pending and the backend sets the next attempt. + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2; + + // A deterministic failure or protocol violation that retrying can never fix. The backend + // preserves the evidence, alerts, and stops automatic retries. + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; +} + +// A stable, bounded reason for a disposition. Never carries a token or raw exception text +// (design §7: tokens expose storage account, container, and blob path). +enum LargePayloadPurgeReason { + LARGE_PAYLOAD_PURGE_REASON_UNSPECIFIED = 0; + + // --- Reported with DELETED --- + + // The blob was deleted by this attempt. + LARGE_PAYLOAD_PURGE_REASON_BLOB_DELETED = 1; + + // The blob was already absent. Deletion is idempotent, so this is a success. + LARGE_PAYLOAD_PURGE_REASON_BLOB_ALREADY_ABSENT = 2; + + // The blob did not carry the payload store's ownership marker, so it was left untouched + // (design §5.5). This is an expected outcome, not a defect: the token text merely matched the + // v2 grammar. The tombstone is still resolved because the blob is not the store's to delete. + LARGE_PAYLOAD_PURGE_REASON_BLOB_NOT_STORE_OWNED = 3; + + // --- Reported with RETRY --- + + // Network failure, timeout, storage outage, throttling, or a 5xx response. + LARGE_PAYLOAD_PURGE_REASON_TRANSIENT_STORAGE_FAILURE = 10; + + // The registered payload store does not implement deletion. Every payload would fail the same + // way, so the work is kept recoverable until an operator registers a store that can delete. + LARGE_PAYLOAD_PURGE_REASON_STORE_CANNOT_DELETE = 11; + + // The token is well formed but points at a storage account this worker's credential cannot + // reach. Recoverable after a configuration or credential change. + LARGE_PAYLOAD_PURGE_REASON_STORAGE_ACCOUNT_UNREACHABLE = 12; + + // The token uses a recognized-but-newer version prefix this worker does not understand. + // Recoverable after an SDK upgrade, so it earns a long defer rather than quarantine. + LARGE_PAYLOAD_PURGE_REASON_UNSUPPORTED_TOKEN_VERSION = 13; + + // Authorization failed in a way that may be transient or reconfigurable (401/403). + LARGE_PAYLOAD_PURGE_REASON_STORAGE_AUTHORIZATION_FAILED = 14; + + // --- Reported with QUARANTINED --- + + // The token carries a known version prefix but its body does not parse. Because the SDK and + // backend control both sides of the protocol, this indicates a producer, corruption, or + // compatibility bug. + LARGE_PAYLOAD_PURGE_REASON_MALFORMED_TOKEN = 20; + + // Storage rejected a request generated from a well-formed token as permanently invalid + // (HTTP 400, e.g. InvalidUri / InvalidResourceName). Retrying can never succeed. + LARGE_PAYLOAD_PURGE_REASON_INVALID_STORAGE_REQUEST = 21; + + // A legacy v1 token reached the worker. This is an invariant violation, because the backend + // excludes v1 at insertion time. It cannot be safely deleted (no storage account in the token) + // and cannot be fixed by retrying, so the evidence is preserved instead (design §6). + LARGE_PAYLOAD_PURGE_REASON_LEGACY_V1_TOKEN = 22; } -// Client -> server. Acknowledges that the worker has deleted the blob for the identified payload so the -// backend can hard-delete the soft-deleted row. -message PayloadPurgeAck { +// client -> server: the outcome of exactly one tombstoned row. +message LargePayloadPurgeResult { + // Row identity, echoed from the corresponding LargePayloadTombstone. int32 partitionId = 1; int64 instanceKey = 2; int64 payloadId = 3; + + // Echoed unmodified from the fetched tombstone; used as a compare-and-swap guard. + int64 revision = 4; + + LargePayloadPurgeDisposition disposition = 5; + LargePayloadPurgeReason reason = 6; + + // Optional bounded, sanitized storage status or error code for diagnostics + // (for example "BlobNotFound" or "409"). Must never contain a token or raw exception text. + string storageErrorCode = 7; } -// Client -> server. Requests up to `limit` tombstoned payloads for the worker to delete. -message GetTombstonedPayloadsRequest { +// client -> server: request up to `limit` due tombstones for the caller's task hub. +message GetLargePayloadTombstonesRequest { + // The maximum number of rows to return. The service clamps this to its own maximum. int32 limit = 1; } -// Server -> client. Carries the batch of tombstoned payloads for the worker to delete. -message GetTombstonedPayloadsResponse { - repeated TombstonedPayload payloads = 1; +// server -> client: the due tombstones whose blobs the worker must delete. +message GetLargePayloadTombstonesResponse { + repeated LargePayloadTombstone tombstones = 1; } -// Client -> server. Acknowledges a batch of payloads whose blobs the worker has deleted. -message AckPurgedPayloadsRequest { - repeated PayloadPurgeAck acks = 1; +// client -> server: a bounded batch of purge outcomes. +message ReportLargePayloadPurgeResultsRequest { + repeated LargePayloadPurgeResult results = 1; } -// Server -> client. Empty response acknowledging the acks were recorded. -message AckPurgedPayloadsResponse { +// server -> client: acknowledgement that the reported outcomes were recorded. +message ReportLargePayloadPurgeResultsResponse { } message GetWorkItemsRequest { @@ -879,6 +971,16 @@ message GetWorkItemsRequest { repeated WorkerCapability capabilities = 10; WorkItemFilters workItemFilters = 11; + + // Task-hub scoped opt-in for large-payload blob auto-purge. + // + // This is a setting, not a WorkerCapability: WORKER_CAPABILITY_LARGE_PAYLOADS means a worker + // *can resolve* externalized payloads, not that the customer opted into *deleting* them. + // A capability list is presence-only and cannot express an explicit false, so a three-state + // wrapper is used instead: + // absent -> the worker has no opinion (older SDK); the persisted setting is unchanged. + // true/false -> an explicit customer choice; persisted only when the value actually differs. + google.protobuf.BoolValue large_payload_auto_purge_enabled = 12; } enum WorkerCapability { diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 5dd18d52..ea5f7c87 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -349,6 +349,10 @@ async ValueTask BuildRuntimeStateAsync( workerOptions.Concurrency.MaximumConcurrentEntityWorkItems, Capabilities = { this.worker.grpcOptions.Capabilities }, WorkItemFilters = this.worker.workItemFilters?.ToGrpcWorkItemFilters(), + + // Left unset when the worker has no opinion, so the backend can distinguish "not + // configured" from an explicit opt-out. + LargePayloadAutoPurgeEnabled = this.worker.grpcOptions.LargePayloadAutoPurgeEnabled, }, cancellationToken: cancellation); } diff --git a/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs b/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs index 59c21a00..a7f5fa34 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs @@ -44,6 +44,13 @@ public sealed class GrpcDurableTaskWorkerOptions : DurableTaskWorkerOptions /// public HashSet Capabilities { get; } = new() { P.WorkerCapability.HistoryStreaming }; + /// + /// Gets or sets a value indicating whether this worker opts in to large-payload auto-purge, announced to + /// the backend on connection. null means the worker expresses no opinion and the backend keeps its + /// current behavior; an explicit true or false is the customer's choice. + /// + public bool? LargePayloadAutoPurgeEnabled { get; set; } + /// /// Gets or sets the maximum size of all actions in a complete orchestration work item chunk. /// The default value is 3.9MB. We leave some headroom to account for request size overhead. diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs new file mode 100644 index 00000000..4a88205e --- /dev/null +++ b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Client.Grpc.Tests; + +/// +/// maps the managed purge enums onto +/// their protobuf counterparts by numeric value rather than by name, which is only correct while the two sides +/// agree on every value. A silent drift would not fail to compile; it would send the backend a different +/// disposition than the worker decided and delete or quarantine the wrong rows. These tests pin the mapping. +/// +public class LargePayloadPurgeEnumParityTests +{ + [Fact] + public void Disposition_ManagedAndProtobufValues_AreIdentical() + { + // Arrange & Act + Dictionary managed = Enum.GetValues(typeof(LargePayloadPurgeDisposition)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + Dictionary proto = Enum.GetValues(typeof(P.LargePayloadPurgeDisposition)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + + // Assert - same numeric values AND the same names at each value, so neither side can gain, lose, or + // renumber a member unnoticed. + managed.Should().Equal(proto); + } + + [Fact] + public void Reason_ManagedAndProtobufValues_AreIdentical() + { + // Arrange & Act + Dictionary managed = Enum.GetValues(typeof(LargePayloadPurgeReason)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + Dictionary proto = Enum.GetValues(typeof(P.LargePayloadPurgeReason)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + + // Assert + managed.Should().Equal(proto); + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 93989325..4002dd36 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -4,88 +4,183 @@ using Azure; using FluentAssertions; using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; using Xunit; namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; public class DeleteExternalBlobActivityTests { + const string V2Token = "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"; + [Fact] - public async Task RunAsync_WhenDeleteThrowsRequestFailed400_DiscardsPoisonToken() + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesWithInvalidStorageRequest() { // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. - StubPayloadStore store = new(new RequestFailedException(400, "InvalidResourceName")); + StubPayloadStore store = new(new RequestFailedException(400, "bad", "InvalidResourceName", null)); DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/bad name"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - discarded so the backend acks and clears the row instead of re-streaming forever. - result.Should().Be(BlobDeleteResult.Discarded); + // Assert - quarantined (evidence preserved), never a success-shaped discard. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + outcome.Reason.Should().Be(LargePayloadPurgeReason.InvalidStorageRequest); + outcome.StorageErrorCode.Should().Be("InvalidResourceName"); } [Fact] - public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_LeavesTombstonedForRetry() + public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_RetriesAsTransient() { - // Arrange - a Status 503 that escaped the SDK's internal retries is treated as transient. - StubPayloadStore store = new(new RequestFailedException(503, "ServerBusy")); + // Arrange - a Status 503 that escaped the SDK's internal retries is still treated as transient. + StubPayloadStore store = new(new RequestFailedException(503, "busy", "ServerBusy", null)); DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - left tombstoned so a later purge cycle can retry; a blob is never dropped on doubt. - result.Should().Be(BlobDeleteResult.Retry); + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TransientStorageFailure); + outcome.StorageErrorCode.Should().Be("ServerBusy"); + } + + [Theory] + [InlineData(401)] + [InlineData(403)] + public async Task RunAsync_WhenDeleteThrowsAuthorizationFailure_Retries(int status) + { + // Arrange - authorization can be fixed by reconfiguration, so it stays recoverable. + StubPayloadStore store = new(new RequestFailedException(status, "denied", "AuthorizationFailure", null)); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageAuthorizationFailed); } [Fact] - public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_DiscardsToUnblockPipeline() + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsAccountUnreachable() { - // Arrange - the payload lives in a storage account the configured credential cannot reach. Retrying can - // never succeed and the backend batch is cursor-less, so a permanently unreachable row would re-stream - // every cycle and block later rows; it must be discarded (acked), not retried. + // Arrange - the payload lives in a storage account the configured credential cannot reach. That is + // recoverable after a configuration or credential change, so it is deferred rather than discarded. StubPayloadStore store = new(new PayloadStorageException("cross-account delete requires identity auth")); DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync( + null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageAccountUnreachable); + } + + [Fact] + public async Task RunAsync_V1Token_QuarantinesWithoutCallingStore() + { + // Arrange - a v1 token names a container but not the storage account, so a delete against the + // configured account cannot be verified and would falsely report success if the store was repointed. + Mock store = new(); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); - // Assert - discarded so the pipeline head-of-line is not blocked by an undeletable payload. - result.Should().Be(BlobDeleteResult.Discarded); + // Assert - quarantined by the gate, and the store's DeleteAsync was never invoked. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + outcome.Reason.Should().Be(LargePayloadPurgeReason.LegacyV1Token); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } [Fact] - public async Task RunAsync_V1Token_DiscardsWithoutCallingStore() + public async Task RunAsync_UnknownTokenVersion_RetriesWithoutCallingStore() { - // Arrange - auto-purge policy: a legacy v1 token identifies no storage account, so it is dropped before - // the store is ever consulted. + // Arrange - an unrecognized prefix most likely came from a newer SDK, which recovers after an upgrade. Mock store = new(); DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v9:https://acct.blob.core.windows.net/c/x"); - // Assert - discarded by the gate, and the store's DeleteAsync was never invoked. - result.Should().Be(BlobDeleteResult.Discarded); + // Assert - retried, NOT quarantined: quarantine is terminal and this can self-heal. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.UnsupportedTokenVersion); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } [Fact] - public async Task RunAsync_V2Token_CallsStore() + public async Task RunAsync_MalformedV2Token_Quarantines() + { + // Arrange - a recognized v2 prefix whose body does not parse; the store signals that with + // ArgumentException. Retrying can never fix a body the SDK itself produced malformed. + StubPayloadStore store = new(new ArgumentException("Invalid token")); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v2:not-a-uri"); + + // Assert - contrast with the unknown-prefix case above, which is retried. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + outcome.Reason.Should().Be(LargePayloadPurgeReason.MalformedToken); + } + + [Fact] + public async Task RunAsync_V2Token_CallsStoreAndReportsDeleted() { // Arrange - a self-describing v2 token is not gated and must reach the store. Mock store = new(); - store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())).Returns(Task.CompletedTask); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.Deleted); DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - the store deleted the blob (proves the gate is v1-only and did not break the happy path). - result.Should().Be(BlobDeleteResult.Deleted); + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobDeleted); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Once); } + [Fact] + public async Task RunAsync_WhenBlobAlreadyAbsent_ReportsDeleted() + { + // Arrange - deletion is idempotent, so a blob a previous attempt already removed is not a failure. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.AlreadyAbsent); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobAlreadyAbsent); + } + + [Fact] + public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeleting() + { + // Arrange - the blob exists but carries no ownership marker, so the store left it untouched. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.NotStoreOwned); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + + // Assert - the tombstone is still resolved: a blob the store does not own is not the store's to delete, + // and re-serving the row forever would never make it deletable. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobNotStoreOwned); + } + [Fact] public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombstone() { @@ -94,12 +189,28 @@ public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombst DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + + // Assert - retried (tombstone preserved): resolving it would destroy the backend's cleanup ledger while + // the blob survives. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StoreCannotDelete); + } + + [Fact] + public async Task RunAsync_WhenDeleteTimesOut_RetriesAsTransient() + { + // Arrange - a non-Azure exception (timeout / network failure) must not drop a blob on doubt. + StubPayloadStore store = new(new TimeoutException()); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - retried (tombstone preserved), never discarded: acking would destroy the backend's cleanup - // ledger while the blob survives. - result.Should().Be(BlobDeleteResult.Retry); - result.Should().NotBe(BlobDeleteResult.Discarded); + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TransientStorageFailure); + outcome.StorageErrorCode.Should().BeNull(); } sealed class StubPayloadStore : PayloadStore @@ -108,8 +219,10 @@ sealed class StubPayloadStore : PayloadStore public StubPayloadStore(Exception? deleteError) => this.deleteError = deleteError; - public override Task DeleteAsync(string token, CancellationToken cancellationToken) => - this.deleteError is null ? Task.CompletedTask : throw this.deleteError; + public override Task DeleteAsync(string token, CancellationToken cancellationToken) => + this.deleteError is null + ? Task.FromResult(PayloadDeleteOutcome.Deleted) + : throw this.deleteError; public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => throw new NotSupportedException(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs b/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs index c03ee76b..4a2111b5 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs @@ -373,6 +373,48 @@ public async Task UploadAsync_StaleContainerNotFoundFailure_DoesNotOverwriteNewe createCalls.Should().Be(2); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task UploadAsync_WritesOwnershipMarkerOnBothWritePaths(bool compressionEnabled) + { + // Arrange - the marker is what lets auto-purge prove the store wrote a blob before deleting it, so it + // must be written on the compressed and the uncompressed path alike. It rides the existing + // BlobOpenWriteOptions, so it costs no extra request. + BlobOpenWriteOptions? capturedOptions = null; + Mock containerClientMock = new(); + containerClientMock.Setup(c => c.Name).Returns("test-container"); + containerClientMock + .Setup(c => c.GetBlobClient(It.IsAny())) + .Returns(() => + { + Mock blobClientMock = new(); + blobClientMock.SetupGet(b => b.Uri).Returns( + new Uri("https://testaccount.blob.core.windows.net/test-container/payload")); + blobClientMock + .Setup(b => b.OpenWriteAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((bool _, BlobOpenWriteOptions options, CancellationToken _) => + { + capturedOptions = options; + return new MemoryStream(); + }); + return blobClientMock.Object; + }); + + LargePayloadStorageOptions options = new() { CompressionEnabled = compressionEnabled }; + BlobPayloadStore store = new(options, containerClientMock.Object); + + // Act + await store.UploadAsync("payload", CancellationToken.None); + + // Assert + capturedOptions.Should().NotBeNull(); + capturedOptions!.Metadata.Should().NotBeNull(); + capturedOptions.Metadata.Should().Contain( + BlobPayloadStore.OwnershipMarkerName, BlobPayloadStore.OwnershipMarkerValue); + } + static Mock CreateContainerClientMock() { Mock containerClientMock = new(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index 05013de4..7ab88c15 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -2,8 +2,11 @@ // Licensed under the MIT License. using FluentAssertions; +using Grpc.Core; using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -115,10 +118,40 @@ public void UseExternalizedPayloads_ClientOnly_RegistersResolvablePayloadStore() // Act - UseDevelopmentStorage=true is a valid connection string that BlobServiceClient accepts with no // network I/O, so the store constructs offline. Build the provider and actually resolve PayloadStore. builder.Object.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); - using ServiceProvider provider = services.BuildServiceProvider(); + using ServiceProvider clientProvider = services.BuildServiceProvider(); // Assert - the store resolves without throwing and is the blob-backed implementation. - PayloadStore store = provider.GetRequiredService(); + PayloadStore store = clientProvider.GetRequiredService(); store.Should().BeOfType(); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void UseExternalizedPayloads_Worker_SendsResolvedAutoPurgeValueOnTheHandshake(bool autoPurge) + { + // Arrange - the backend only tombstones payloads for a task hub whose workers opted in, so the resolved + // AutoPurge value has to ride the GetWorkItems handshake. A worker that never calls this extension sends + // nothing at all, which the backend reads as "no opinion". + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + services.AddOptions(string.Empty) + .Configure(o => o.CallInvoker = Mock.Of()); + + // Act + builder.Object.UseExternalizedPayloads(options => + { + options.ConnectionString = "UseDevelopmentStorage=true"; + options.AutoPurge = autoPurge; + }); + + using ServiceProvider provider = services.BuildServiceProvider(); + GrpcDurableTaskWorkerOptions grpcOptions = + provider.GetRequiredService>().Get(string.Empty); + + // Assert - an explicit true/false, never left absent, because calling this extension IS the choice. + grpcOptions.LargePayloadAutoPurgeEnabled.Should().Be(autoPurge); + } } diff --git a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs index 7ccab24a..30c7cee3 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs @@ -9,14 +9,17 @@ namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests; /// -/// Unit tests for , covering legacy v1 back-compatibility and the -/// self-describing v2 token resolution (same account, cross-account with identity, cross-account without). +/// Unit tests for , covering legacy v1 back-compatibility, the +/// self-describing v2 token resolution (same account, cross-account with identity, cross-account without), and +/// the ownership marker that gates every delete. /// public class BlobPayloadStoreDeleteTests { const string ContainerName = "payloads"; const string ConfiguredAccountUrl = "https://myaccount.blob.core.windows.net"; + static readonly ETag KnownETag = new("\"0x8DTEST\""); + static Mock CreateContainer(Mock blob, string expectedBlobName) { Mock container = new(); @@ -26,9 +29,34 @@ static Mock CreateContainer(Mock blob, string e return container; } - static Mock CreateBlob(bool existed) + /// + /// Creates a blob that exists and carries this store's ownership marker, which is the ordinary case for a + /// payload the store itself uploaded. + /// + static Mock CreateBlob(bool existed) => CreateBlob(existed, owned: true); + + static Mock CreateBlob(bool existed, bool owned) { Mock blob = new(); + + if (existed) + { + Dictionary metadata = owned + ? new() { [BlobPayloadStore.OwnershipMarkerName] = BlobPayloadStore.OwnershipMarkerValue } + : new() { ["customer-tag"] = "not-ours" }; + + blob + .Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue( + BlobsModelFactory.BlobProperties(metadata: metadata, eTag: KnownETag), Mock.Of())); + } + else + { + blob + .Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(404, "not found", "BlobNotFound", null)); + } + blob .Setup(b => b.DeleteIfExistsAsync( It.IsAny(), It.IsAny(), It.IsAny())) @@ -45,17 +73,22 @@ public async Task DeleteAsync_V1Token_DeletesBackingBlobIncludingSnapshots() BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); // Act - await store.DeleteAsync($"blob:v1:{ContainerName}:abc123", CancellationToken.None); + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v1:{ContainerName}:abc123", CancellationToken.None); // Assert + outcome.Should().Be(PayloadDeleteOutcome.Deleted); container.Verify(c => c.GetBlobClient("abc123"), Times.Once); blob.Verify( - b => b.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, It.IsAny()), + b => b.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + It.Is(c => c.IfMatch == KnownETag), + It.IsAny()), Times.Once); } [Fact] - public async Task DeleteAsync_MissingBlob_IsIdempotentAndDoesNotThrow() + public async Task DeleteAsync_MissingBlob_IsIdempotentAndReportsAlreadyAbsent() { // Arrange Mock blob = CreateBlob(existed: false); @@ -63,13 +96,36 @@ public async Task DeleteAsync_MissingBlob_IsIdempotentAndDoesNotThrow() BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); // Act (a missing blob must be a no-op, not an error) - await store.DeleteAsync($"blob:v1:{ContainerName}:missing", CancellationToken.None); + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v1:{ContainerName}:missing", CancellationToken.None); - // Assert + // Assert - the ownership probe already proved absence, so no delete request is needed. + outcome.Should().Be(PayloadDeleteOutcome.AlreadyAbsent); blob.Verify( b => b.DeleteIfExistsAsync( It.IsAny(), It.IsAny(), It.IsAny()), - Times.Once); + Times.Never); + } + + [Fact] + public async Task DeleteAsync_BlobWithoutOwnershipMarker_LeavesBlobUntouched() + { + // Arrange - a blob whose token text matches the v2 grammar but which this store never wrote (for + // example a customer dataset referenced by URL, or a payload written before the marker shipped). + Mock blob = CreateBlob(existed: true, owned: false); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); + + // Act + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v2:{ConfiguredAccountUrl}/{ContainerName}/abc123", CancellationToken.None); + + // Assert - reported distinctly and, critically, never deleted. + outcome.Should().Be(PayloadDeleteOutcome.NotStoreOwned); + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); } [Fact] @@ -115,12 +171,17 @@ public async Task DeleteAsync_V2TokenSameContainer_DeletesViaConfiguredClient() BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); // Act - await store.DeleteAsync($"blob:v2:{ConfiguredAccountUrl}/{ContainerName}/abc123", CancellationToken.None); + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v2:{ConfiguredAccountUrl}/{ContainerName}/abc123", CancellationToken.None); // Assert + outcome.Should().Be(PayloadDeleteOutcome.Deleted); container.Verify(c => c.GetBlobClient("abc123"), Times.Once); blob.Verify( - b => b.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, It.IsAny()), + b => b.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + It.Is(c => c.IfMatch == KnownETag), + It.IsAny()), Times.Once); } From 2026d71bf533fafbce1a618789eea64ea24c1145 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 15:33:28 -0700 Subject: [PATCH 19/47] Clear three new-code warnings in the auto-purge files Pre-review polish only; no behaviour change. CS8603 in DeleteExternalBlobActivity.SanitizeErrorCode fired on netstandard2.0 only, where string.IsNullOrEmpty carries no [NotNullWhen(false)] annotation so flow analysis cannot prove the else branch non-null. Replaced with a constant pattern, which the compiler analyses itself and so behaves identically on every target framework. CA1001 in BlobPurgeJobStarter: the type owned a CancellationTokenSource but was not disposable. The source is deliberately disposed in Dispose rather than in StopAsync -- StopAsync stops waiting as soon as the host shutdown token fires, so the ensure task may still hold the token, and disposing there would fault that still-running task with ObjectDisposedException. The container disposes singletons after every StopAsync has returned, which is the safe point. SA1600: documented the BlobPurgeJobStarter constructor. CA1873 on BlobPurgeJobOrchestrator is deliberately left as-is: it occurs 77 times across this repo, so unguarded log calls are the established local convention and changing one new file would make it inconsistent with its neighbours for no benefit. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Activities/DeleteExternalBlobActivity.cs | 12 ++++++--- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 25 ++++++++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index e811f46f..b86a0727 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -81,10 +81,16 @@ public override async Task RunAsync(TaskActivityContext contex /// example BlobNotFound) is a fixed vocabulary and the numeric status is the fallback, so neither /// can carry a token or raw exception text. /// - static string SanitizeErrorCode(RequestFailedException exception) => - string.IsNullOrEmpty(exception.ErrorCode) + static string SanitizeErrorCode(RequestFailedException exception) + { + // Pattern-matched rather than string.IsNullOrEmpty: on netstandard2.0 that method carries no + // [NotNullWhen(false)] annotation, so flow analysis cannot prove the else branch is non-null and warns. + // A constant pattern is analyzed by the compiler itself and so behaves the same on every target. + string? errorCode = exception.ErrorCode; + return errorCode is null or "" ? exception.Status.ToString(CultureInfo.InvariantCulture) - : exception.ErrorCode; + : errorCode; + } async Task DeleteAsync(string token) { diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index b6f10d9c..3508120d 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -18,7 +18,7 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// that retries until the backend is reachable. The job is a per-task-hub singleton, so racing client /// processes simply no-op. /// -sealed class BlobPurgeJobStarter : IHostedService +sealed class BlobPurgeJobStarter : IHostedService, IDisposable { static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); @@ -32,6 +32,14 @@ sealed class BlobPurgeJobStarter : IHostedService CancellationTokenSource? cts; Task? ensureTask; + /// + /// Initializes a new instance of the class. + /// + /// The provider used to resolve the named durable task client. + /// The registered payload store, checked for delete support before starting the job. + /// The monitor used to read the fully-resolved large payload storage options. + /// The name of the client builder this starter belongs to. + /// The logger. public BlobPurgeJobStarter( IDurableTaskClientProvider clientProvider, PayloadStore store, @@ -100,6 +108,21 @@ public async Task StopAsync(CancellationToken cancellationToken) } } + /// + /// Disposes the cancellation source backing the background ensure task. + /// + /// + /// Deliberately not disposed in : that method stops waiting as soon as the host's + /// shutdown token fires, so the ensure task may still hold this source's token. Disposing it there would + /// fault that still-running task with an when it next registers a + /// callback. The container disposes singletons after every has returned, which is + /// the safe point. + /// + public void Dispose() + { + this.cts?.Dispose(); + } + async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) From e25e72437b0c50787b05c51bce56690d8db4c8e2 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 15:37:32 -0700 Subject: [PATCH 20/47] Pin the no-inbound-enum invariant the numeric purge casts rely on The managed-to-proto enum casts in ReportLargePayloadPurgeResultsAsync are safe only because no enum crosses the wire inbound on this feature: the SDK casts values it defined itself, so it can never receive an unknown value and silently reinterpret it. That invariant held by the shape of the contract rather than by construction, and nothing in the code stated it. Adding an enum to an inbound type would create exactly the hazard the casts avoid today -- a newer backend sending a value this SDK does not know, mapped by raw numeric cast onto a valid-but-wrong member -- and it would compile silently, mis-dispositioning rows in production rather than failing a build. Asserted by reflection over the inbound types in the existing parity test file, which already exists to protect this mapping. The casts are unchanged: they are correct today, and an explicit switch would add branching for a hazard that does not currently exist on this path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../LargePayloadPurgeEnumParityTests.cs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs index 4a88205e..bd563897 100644 --- a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs +++ b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Reflection; using Microsoft.DurableTask.Client; using P = Microsoft.DurableTask.Protobuf; @@ -44,4 +45,36 @@ public void Reason_ManagedAndProtobufValues_AreIdentical() // Assert managed.Should().Equal(proto); } + + /// + /// The numeric casts are safe only because no enum crosses the wire inbound on this feature: the SDK + /// casts values it defined itself, so it can never receive an unknown value and silently reinterpret it. + /// That invariant holds today by the shape of the contract, not by construction, and nothing in the code + /// states it. Adding an enum to an inbound type would create exactly that path - a newer backend sending a + /// value this SDK does not know, mapped by raw numeric cast onto a valid-but-wrong member - and it would + /// compile silently. This pins the invariant so it fails here instead. + /// + /// A type carrying server-to-client data for the purge feature. + [Theory] + [InlineData(typeof(LargePayloadTombstone))] + [InlineData(typeof(P.LargePayloadTombstone))] + [InlineData(typeof(P.GetLargePayloadTombstonesResponse))] + [InlineData(typeof(P.ReportLargePayloadPurgeResultsResponse))] + public void InboundTypes_ExposeNoEnumMembers(Type inboundType) + { + // Arrange & Act + List enumMembers = inboundType + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => (Nullable.GetUnderlyingType(p.PropertyType) ?? p.PropertyType).IsEnum) + .Select(p => $"{inboundType.Name}.{p.Name}") + .ToList(); + + // Assert + enumMembers.Should().BeEmpty( + "an enum on an inbound type invalidates the numeric enum casts in " + + "GrpcDurableTaskClient.ReportLargePayloadPurgeResultsAsync. The SDK would map a value chosen by the " + + "backend - including one a newer backend added that this SDK does not know - onto a managed member " + + "by raw numeric value, silently mis-dispositioning rows. Map inbound enums explicitly instead, with " + + "a switch that handles unknown values"); + } } From 7bf5da8f08e4e6db6d81ebc171ba905664c0581b Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 15:41:25 -0700 Subject: [PATCH 21/47] Clear the two remaining PR-introduced style warnings CA1859: CreateOwnershipMetadata returned IDictionary while always constructing a Dictionary. The method is private static with two call sites, both assigning to BlobOpenWriteOptions.Metadata (IDictionary), so returning the concrete type is implicitly compatible and lets the JIT devirtualize the indexer. SA1117: split the ArgumentOutOfRangeException arguments one per line. CA1873 in BlobPurgeJobOrchestrator is left deliberately: unguarded logging is the established convention here (77 occurrences solution-wide), so guarding one new file would make it inconsistent with its neighbours. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AzureBlobPayloads/Options/LargePayloadStorageOptions.cs | 3 ++- .../AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 2b0b807a..49cb020d 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -147,7 +147,8 @@ public int PayloadPurgeBatchSize if (value < 1 || value > BlobPurgeConstants.MaxBatchSize) { throw new ArgumentOutOfRangeException( - nameof(this.PayloadPurgeBatchSize), value, + nameof(this.PayloadPurgeBatchSize), + value, $"PayloadPurgeBatchSize must be between 1 and {BlobPurgeConstants.MaxBatchSize} (inclusive)."); } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index 9a22cd03..bc597e4c 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -374,7 +374,7 @@ internal static DecodeTokenResult DecodeToken(string token) /// Creates the ownership metadata stamped on every blob this store writes, so a later purge can prove the /// store created the blob before deleting it. /// - static IDictionary CreateOwnershipMetadata() => + static Dictionary CreateOwnershipMetadata() => new Dictionary(StringComparer.OrdinalIgnoreCase) { [OwnershipMarkerName] = OwnershipMarkerValue, From de218c84d76189c4511a852cf3e308303810f902 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 16:08:49 -0700 Subject: [PATCH 22/47] Narrow the purge reason enum from 11 values to 7 The backend never branches on reason - it acts on disposition alone - so reason is diagnostics only, and its granularity should match the number of distinct operator responses rather than the number of distinct causes. storageErrorCode already carries the specific storage status, so four of the old values encoded the same fact twice. TRANSIENT_STORAGE_FAILURE, STORAGE_ACCOUNT_UNREACHABLE and STORAGE_AUTHORIZATION_FAILED collapse into STORAGE_FAILURE. MALFORMED_TOKEN, INVALID_STORAGE_REQUEST, LEGACY_V1_TOKEN and UNSUPPORTED_TOKEN_VERSION collapse into TOKEN_NOT_PURGEABLE. STORE_CANNOT_DELETE stays separate because storage is never contacted, so its storageErrorCode is empty. Dispositions are unchanged on every branch. TOKEN_NOT_PURGEABLE now spans both Quarantined and Retry - an unsupported version prefix stays Retry because an SDK upgrade resolves it - so each branch continues to state its disposition explicitly rather than deriving it from the reason, which would strand those rows permanently. Proto verified byte-identical to contract-canonical.proto across all three repos by the marker-based harness. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- src/Client/Core/LargePayloadPurgeReason.cs | 84 +++++++---------- .../Activities/DeleteExternalBlobActivity.cs | 20 +++-- src/Grpc/orchestrator_service.proto | 89 ++++++++++--------- .../DeleteExternalBlobActivityTests.cs | 29 +++--- 4 files changed, 106 insertions(+), 116 deletions(-) diff --git a/src/Client/Core/LargePayloadPurgeReason.cs b/src/Client/Core/LargePayloadPurgeReason.cs index 4be5190d..c957bad0 100644 --- a/src/Client/Core/LargePayloadPurgeReason.cs +++ b/src/Client/Core/LargePayloadPurgeReason.cs @@ -4,9 +4,13 @@ namespace Microsoft.DurableTask.Client; /// -/// A stable, bounded reason for a . Never carries a token or raw -/// exception text, because tokens expose the storage account, container, and blob path. Mirrors the -/// LargePayloadPurgeReason protobuf enum. +/// Why a row received its . Diagnostic only: the backend acts on the +/// disposition alone and never branches on this value, so it exists to make a stuck or non-reclaiming ledger +/// explainable without access to worker logs. Deliberately coarse - granularity matches the number of distinct +/// operator responses, not the number of distinct causes, because +/// already carries the specific storage status. Never +/// carries a token or raw exception text, because a token exposes the storage account, container, and blob path. +/// Mirrors the LargePayloadPurgeReason protobuf enum. /// public enum LargePayloadPurgeReason { @@ -16,76 +20,52 @@ public enum LargePayloadPurgeReason Unspecified = 0, /// - /// The blob was deleted by this attempt. Reported with . + /// The blob was deleted by this attempt, reclaiming its bytes. Reported with + /// . /// BlobDeleted = 1, /// - /// The blob was already absent. Deletion is idempotent, so this is a success. Reported with + /// The blob was already absent, so deletion was a no-op and no bytes were reclaimed. Deletion is idempotent, + /// so this is a success rather than a failure; it is reported separately from + /// because a high rate of it indicates duplicate tombstones. Reported with /// . /// BlobAlreadyAbsent = 2, /// /// The blob did not carry the payload store's ownership marker, so it was left untouched. This is an - /// expected outcome, not a defect: the token text merely matched the v2 grammar. The tombstone is still - /// resolved because the blob is not the store's to delete. Reported with - /// . + /// expected outcome, not a defect: the token text merely matched the v2 grammar, and the payload column is + /// customer-writable. The tombstone is still resolved, because a blob the store does not own will never + /// become deletable. Reported separately from so that "resolved without + /// reclaiming bytes" stays countable. Reported with . /// BlobNotStoreOwned = 3, /// - /// Network failure, timeout, storage outage, throttling, or a 5xx response. Reported with - /// . + /// The deletion failed against storage: network failure, timeout, outage, throttling, an unreachable + /// account, or an authorization failure. All of these are reconfigurable or self-healing, and they are not + /// subdivided because already carries the specific + /// status. Reported with . /// - TransientStorageFailure = 10, + StorageFailure = 10, /// - /// The registered payload store does not implement deletion. Every payload would fail the same way, so the - /// work is kept recoverable until an operator registers a store that can delete. Reported with + /// The registered payload store does not implement deletion. Every payload fails the same way, so this is a + /// deployment-wide condition rather than a per-row one, and it stays recoverable until an operator registers + /// a store that can delete. is empty because storage + /// was never contacted, which is why this is not folded into . Reported with /// . /// StoreCannotDelete = 11, /// - /// The token is well formed but points at a storage account this worker's credential cannot reach. - /// Recoverable after a configuration or credential change. Reported with - /// . - /// - StorageAccountUnreachable = 12, - - /// - /// The token uses a recognized-but-newer version prefix this worker does not understand. Recoverable after - /// an SDK upgrade, so it earns a long defer rather than quarantine. Reported with - /// . - /// - UnsupportedTokenVersion = 13, - - /// - /// Authorization failed in a way that may be transient or reconfigurable (401/403). Reported with - /// . - /// - StorageAuthorizationFailed = 14, - - /// - /// The token carries a known version prefix but its body does not parse. Because the SDK and backend - /// control both sides of the protocol, this indicates a producer, corruption, or compatibility bug. - /// Reported with . - /// - MalformedToken = 20, - - /// - /// Storage rejected a request generated from a well-formed token as permanently invalid (HTTP 400, for - /// example InvalidUri / InvalidResourceName). Retrying can never succeed. Reported with - /// . - /// - InvalidStorageRequest = 21, - - /// - /// A legacy v1 token reached the worker. This is an invariant violation, because the backend excludes v1 - /// at insertion time. It cannot be safely deleted (no storage account in the token) and cannot be fixed by - /// retrying, so the evidence is preserved instead. Reported with - /// . + /// The token cannot be acted on as it stands: its body does not parse, it names a version this worker does + /// not support, or storage rejected it as permanently invalid. + /// distinguishes the storage-rejected case, where it + /// is populated, from the parse cases, where it is empty. Reported with + /// , except for an unsupported version prefix, which + /// is reported with because an SDK upgrade resolves it. /// - LegacyV1Token = 22, + TokenNotPurgeable = 20, } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index b86a0727..7304d4db 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -107,15 +107,17 @@ async Task DeleteAsync(string token) // its token as evidence and stops polling it. The backend excludes v1 at insertion time, so // reaching this branch is an invariant violation rather than an expected path. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.LegacyV1Token); + LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.TokenNotPurgeable); } if (!token.StartsWith(BlobPayloadStore.TokenPrefixV2, StringComparison.Ordinal)) { // An unrecognized prefix is most likely a token written by a newer SDK than this worker runs. That - // recovers after an upgrade, so it earns a deferral rather than quarantine. + // recovers after an upgrade, so it earns a deferral rather than quarantine. This branch shares + // TokenNotPurgeable with the quarantined token cases, so its disposition is deliberately stated + // here rather than derived from the reason: folding it in would strand rows an upgrade would fix. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.UnsupportedTokenVersion); + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.TokenNotPurgeable); } try @@ -142,7 +144,7 @@ async Task DeleteAsync(string token) // body that does not parse. The SDK and backend control both sides of the protocol, so that // indicates a producer, corruption, or compatibility bug; retrying can never fix it. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.MalformedToken); + LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.TokenNotPurgeable); } catch (NotSupportedException) { @@ -157,7 +159,7 @@ async Task DeleteAsync(string token) // (account-key auth is account-specific). Recoverable after a configuration or credential change, // so it is deferred rather than discarded. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageAccountUnreachable); + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageFailure); } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.BadRequest) { @@ -165,7 +167,7 @@ async Task DeleteAsync(string token) // example InvalidUri / InvalidResourceName). Retrying can never succeed. return new BlobPurgeOutcome( LargePayloadPurgeDisposition.Quarantined, - LargePayloadPurgeReason.InvalidStorageRequest, + LargePayloadPurgeReason.TokenNotPurgeable, SanitizeErrorCode(ex)); } catch (RequestFailedException ex) when ( @@ -175,7 +177,7 @@ async Task DeleteAsync(string token) // dropping data an operator can still reclaim. return new BlobPurgeOutcome( LargePayloadPurgeDisposition.Retry, - LargePayloadPurgeReason.StorageAuthorizationFailed, + LargePayloadPurgeReason.StorageFailure, SanitizeErrorCode(ex)); } catch (RequestFailedException ex) @@ -184,14 +186,14 @@ async Task DeleteAsync(string token) // ownership check: transient by default. return new BlobPurgeOutcome( LargePayloadPurgeDisposition.Retry, - LargePayloadPurgeReason.TransientStorageFailure, + LargePayloadPurgeReason.StorageFailure, SanitizeErrorCode(ex)); } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // Timeouts, cancellation, and network failures. A blob is never dropped on an uncertain error. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.TransientStorageFailure); + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageFailure); } } } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index fc007e9a..02ec0aee 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -793,9 +793,10 @@ service TaskHubSidecarService { // pending until its outcome is reported, so this is safe under retries and duplicate callers. rpc GetLargePayloadTombstones(GetLargePayloadTombstonesRequest) returns (GetLargePayloadTombstonesResponse); - // Reports the outcome of each attempted blob deletion. The backend owns retry scheduling: it - // deletes rows reported as DELETED, reschedules RETRY with a reason-appropriate next attempt, - // and moves QUARANTINED rows out of the active fetch while preserving their evidence. + // Reports the outcome of each attempted blob deletion. The backend owns retry scheduling and + // branches solely on `disposition`: it deletes rows reported as DELETED, reschedules RETRY on + // its own backoff, and moves QUARANTINED rows out of the active fetch while preserving their + // evidence. The worker never computes a retry delay. rpc ReportLargePayloadPurgeResults(ReportLargePayloadPurgeResultsRequest) returns (ReportLargePayloadPurgeResultsResponse); rpc CompleteActivityTask(ActivityResponse) returns (CompleteTaskResponse); @@ -846,7 +847,7 @@ message LargePayloadTombstone { // A self-describing SDK v2 token: "blob:v2:{fullBlobUrl}". // Legacy v1 tokens are never tombstoned: v1 carries a container name but not the storage // account, so a delete against the configured account cannot be verified. The backend - // hard-deletes v1 payload rows instead (design §8). + // hard-deletes v1 payload rows instead. string token = 4; // Optimistic-concurrency guard. The worker echoes this value back unmodified in the result so @@ -856,11 +857,16 @@ message LargePayloadTombstone { // The outcome of a single blob deletion attempt. The split is by whether a failure can self-heal. enum LargePayloadPurgeDisposition { + // Required: proto3 reserves 0 as the first value, and scalars have no field presence, so an + // unset field arrives as 0. Keeping 0 meaningless is load-bearing here: if 0 meant DELETED, a + // client that failed to set this field would make the backend delete tombstones and orphan the + // blobs permanently. The backend must reject a result carrying this value. LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; - // Terminal success. The blob was deleted, was already absent, or was deliberately left in - // place because it is not owned by the payload store (design §5.5). The backend deletes the - // tombstone in all three cases. + // Terminal success: the tombstone is resolved and the backend deletes it. Covers the blob being + // deleted, the blob already being absent, and the blob being deliberately left in place because + // the payload store does not own it. All three are terminal because none of them can be + // improved by trying again. LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; // The failure may self-heal, so the row stays pending and the backend sets the next attempt. @@ -871,59 +877,58 @@ enum LargePayloadPurgeDisposition { LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; } -// A stable, bounded reason for a disposition. Never carries a token or raw exception text -// (design §7: tokens expose storage account, container, and blob path). +// Why a row received its disposition. Diagnostic only: the backend acts on `disposition` alone and +// never branches on this value, so it exists to make a stuck or non-reclaiming ledger explainable +// without access to worker logs, which run in the customer's process. +// +// Deliberately coarse. Granularity matches the number of distinct operator responses, not the +// number of distinct causes, because `storageErrorCode` already carries the specific storage status. +// Values must never carry a token or raw exception text: a token exposes the storage account, +// container, and blob path. enum LargePayloadPurgeReason { LARGE_PAYLOAD_PURGE_REASON_UNSPECIFIED = 0; // --- Reported with DELETED --- - // The blob was deleted by this attempt. + // The blob was deleted by this attempt, reclaiming its bytes. LARGE_PAYLOAD_PURGE_REASON_BLOB_DELETED = 1; - // The blob was already absent. Deletion is idempotent, so this is a success. + // The blob was already absent, so deletion was a no-op and no bytes were reclaimed by this + // attempt. Deletion is idempotent, so this is a success rather than a failure. Reported + // separately from BLOB_DELETED because a high rate of it indicates duplicate tombstones. LARGE_PAYLOAD_PURGE_REASON_BLOB_ALREADY_ABSENT = 2; - // The blob did not carry the payload store's ownership marker, so it was left untouched - // (design §5.5). This is an expected outcome, not a defect: the token text merely matched the - // v2 grammar. The tombstone is still resolved because the blob is not the store's to delete. + // The blob was left in place because it did not carry the payload store's ownership marker, + // meaning the store did not write it. The token text merely matched the v2 grammar; the payload + // column is customer-writable, so matching text is not proof of ownership. This is an expected + // outcome rather than a defect. The tombstone is still resolved, because a blob the store does + // not own will never become deletable and retrying forever would leak the row. Reported + // separately from BLOB_DELETED so that "resolved without reclaiming bytes" stays countable. LARGE_PAYLOAD_PURGE_REASON_BLOB_NOT_STORE_OWNED = 3; // --- Reported with RETRY --- - // Network failure, timeout, storage outage, throttling, or a 5xx response. - LARGE_PAYLOAD_PURGE_REASON_TRANSIENT_STORAGE_FAILURE = 10; + // The deletion failed against storage: network failure, timeout, outage, throttling, an + // unreachable account, or an authorization failure. All of these are reconfigurable or + // self-healing, and they are not subdivided here because `storageErrorCode` already carries the + // specific status for diagnostics. Subdividing would encode the same fact twice. + LARGE_PAYLOAD_PURGE_REASON_STORAGE_FAILURE = 10; - // The registered payload store does not implement deletion. Every payload would fail the same - // way, so the work is kept recoverable until an operator registers a store that can delete. + // The registered payload store does not implement deletion. Every payload fails the same way, + // so this is a deployment-wide condition rather than a per-row one, and it is kept recoverable + // until an operator registers a store that can delete. `storageErrorCode` is empty because + // storage was never contacted, which is why this cannot be folded into STORAGE_FAILURE. LARGE_PAYLOAD_PURGE_REASON_STORE_CANNOT_DELETE = 11; - // The token is well formed but points at a storage account this worker's credential cannot - // reach. Recoverable after a configuration or credential change. - LARGE_PAYLOAD_PURGE_REASON_STORAGE_ACCOUNT_UNREACHABLE = 12; - - // The token uses a recognized-but-newer version prefix this worker does not understand. - // Recoverable after an SDK upgrade, so it earns a long defer rather than quarantine. - LARGE_PAYLOAD_PURGE_REASON_UNSUPPORTED_TOKEN_VERSION = 13; - - // Authorization failed in a way that may be transient or reconfigurable (401/403). - LARGE_PAYLOAD_PURGE_REASON_STORAGE_AUTHORIZATION_FAILED = 14; - // --- Reported with QUARANTINED --- - // The token carries a known version prefix but its body does not parse. Because the SDK and - // backend control both sides of the protocol, this indicates a producer, corruption, or - // compatibility bug. - LARGE_PAYLOAD_PURGE_REASON_MALFORMED_TOKEN = 20; - - // Storage rejected a request generated from a well-formed token as permanently invalid - // (HTTP 400, e.g. InvalidUri / InvalidResourceName). Retrying can never succeed. - LARGE_PAYLOAD_PURGE_REASON_INVALID_STORAGE_REQUEST = 21; - - // A legacy v1 token reached the worker. This is an invariant violation, because the backend - // excludes v1 at insertion time. It cannot be safely deleted (no storage account in the token) - // and cannot be fixed by retrying, so the evidence is preserved instead (design §6). - LARGE_PAYLOAD_PURGE_REASON_LEGACY_V1_TOKEN = 22; + // The token cannot be acted on and no retry can change that: its body does not parse, it names + // a version this worker does not support, or storage rejected it as permanently invalid. The + // producer and consumer of this token are both controlled by the SDK and backend, so reaching + // this state indicates a producer, corruption, or compatibility bug and the evidence is + // preserved for investigation rather than discarded. `storageErrorCode` distinguishes the + // storage-rejected case, where it is populated, from the parse cases, where it is empty. + LARGE_PAYLOAD_PURGE_REASON_TOKEN_NOT_PURGEABLE = 20; } // client -> server: the outcome of exactly one tombstoned row. diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 4002dd36..7949cc57 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -14,7 +14,7 @@ public class DeleteExternalBlobActivityTests const string V2Token = "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"; [Fact] - public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesWithInvalidStorageRequest() + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesAsTokenNotPurgeable() { // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. StubPayloadStore store = new(new RequestFailedException(400, "bad", "InvalidResourceName", null)); @@ -23,9 +23,10 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesWithInval // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - quarantined (evidence preserved), never a success-shaped discard. + // Assert - quarantined (evidence preserved), never a success-shaped discard. The storage error code + // is what distinguishes this from the parse-failure cases, which share the same reason. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.InvalidStorageRequest); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); outcome.StorageErrorCode.Should().Be("InvalidResourceName"); } @@ -41,7 +42,7 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_RetriesAsTransien // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TransientStorageFailure); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); outcome.StorageErrorCode.Should().Be("ServerBusy"); } @@ -59,11 +60,11 @@ public async Task RunAsync_WhenDeleteThrowsAuthorizationFailure_Retries(int stat // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageAuthorizationFailed); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); } [Fact] - public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsAccountUnreachable() + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsStorageFailure() { // Arrange - the payload lives in a storage account the configured credential cannot reach. That is // recoverable after a configuration or credential change, so it is deferred rather than discarded. @@ -76,7 +77,7 @@ public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsAcco // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageAccountUnreachable); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); } [Fact] @@ -92,7 +93,7 @@ public async Task RunAsync_V1Token_QuarantinesWithoutCallingStore() // Assert - quarantined by the gate, and the store's DeleteAsync was never invoked. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.LegacyV1Token); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -106,9 +107,11 @@ public async Task RunAsync_UnknownTokenVersion_RetriesWithoutCallingStore() // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v9:https://acct.blob.core.windows.net/c/x"); - // Assert - retried, NOT quarantined: quarantine is terminal and this can self-heal. + // Assert - retried, NOT quarantined: quarantine is terminal and this can self-heal. This is the one + // row where TokenNotPurgeable does not mean Quarantined, so the disposition is asserted deliberately: + // deriving it from the reason would strand rows that an SDK upgrade would have resolved. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.UnsupportedTokenVersion); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -123,9 +126,9 @@ public async Task RunAsync_MalformedV2Token_Quarantines() // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v2:not-a-uri"); - // Assert - contrast with the unknown-prefix case above, which is retried. + // Assert - contrast with the unknown-prefix case above, which shares this reason but is retried. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.MalformedToken); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); } [Fact] @@ -209,7 +212,7 @@ public async Task RunAsync_WhenDeleteTimesOut_RetriesAsTransient() // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TransientStorageFailure); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); outcome.StorageErrorCode.Should().BeNull(); } From 02b957ce6f198392827d10f37bbce08ae5d88afd Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 16:18:07 -0700 Subject: [PATCH 23/47] Re-sync purge reason comments from canonical contract Comment-only. No enum value, field number, disposition, or code path changes; a non-comment-line filter over the full diff of both files returns empty. Block 3 of orchestrator_service.proto is re-spliced verbatim from the canonical contract. The "--- Reported with DELETED / RETRY / QUARANTINED ---" banners are removed: they asserted a 1:1 reason-to-disposition mapping that does not hold, because TOKEN_NOT_PURGEABLE is deliberately reported with two dispositions (QUARANTINED for a v1/malformed/invalid-request token, RETRY for an unrecognized version prefix that a newer worker can read). The disposition now lives in each value's own comment, and the enum header states outright that reason and disposition are orthogonal and that no mapping between them may be asserted. LargePayloadPurgeReason XML docs are rewritten to mirror the new canonical prose, so the managed surface no longer describes the superseded grouping. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- src/Client/Core/LargePayloadPurgeReason.cs | 69 ++++++++++++++-------- src/Grpc/orchestrator_service.proto | 69 ++++++++++++---------- 2 files changed, 83 insertions(+), 55 deletions(-) diff --git a/src/Client/Core/LargePayloadPurgeReason.cs b/src/Client/Core/LargePayloadPurgeReason.cs index c957bad0..4984bf35 100644 --- a/src/Client/Core/LargePayloadPurgeReason.cs +++ b/src/Client/Core/LargePayloadPurgeReason.cs @@ -10,6 +10,12 @@ namespace Microsoft.DurableTask.Client; /// operator responses, not the number of distinct causes, because /// already carries the specific storage status. Never /// carries a token or raw exception text, because a token exposes the storage account, container, and blob path. +/// +/// Reason and disposition are orthogonal and there is no fixed mapping between them. A reason says what the +/// worker found; a disposition says whether trying again can change it. Most reasons occur with exactly one +/// disposition, but deliberately occurs with two. Do not assert a +/// reason-to-disposition mapping anywhere. +/// /// Mirrors the LargePayloadPurgeReason protobuf enum. /// public enum LargePayloadPurgeReason @@ -20,52 +26,65 @@ public enum LargePayloadPurgeReason Unspecified = 0, /// - /// The blob was deleted by this attempt, reclaiming its bytes. Reported with - /// . + /// Reported with . The blob was deleted by this attempt, + /// reclaiming its bytes. /// BlobDeleted = 1, /// - /// The blob was already absent, so deletion was a no-op and no bytes were reclaimed. Deletion is idempotent, - /// so this is a success rather than a failure; it is reported separately from - /// because a high rate of it indicates duplicate tombstones. Reported with - /// . + /// Reported with . The blob was already absent, so deletion + /// was a no-op and no bytes were reclaimed by this attempt. Deletion is idempotent, so this is a success + /// rather than a failure. Reported separately from because a high rate of it + /// indicates duplicate tombstones. /// BlobAlreadyAbsent = 2, /// - /// The blob did not carry the payload store's ownership marker, so it was left untouched. This is an - /// expected outcome, not a defect: the token text merely matched the v2 grammar, and the payload column is - /// customer-writable. The tombstone is still resolved, because a blob the store does not own will never - /// become deletable. Reported separately from so that "resolved without - /// reclaiming bytes" stays countable. Reported with . + /// Reported with . The blob was left in place because it + /// did not carry the payload store's ownership marker, meaning the store did not write it. The token text + /// merely matched the v2 grammar; the payload column is customer-writable, so matching text is not proof of + /// ownership. This is an expected outcome rather than a defect. The tombstone is still resolved, because a + /// blob the store does not own will never become deletable and retrying forever would leak the row. Reported + /// separately so that "resolved without reclaiming bytes" stays countable. /// BlobNotStoreOwned = 3, /// - /// The deletion failed against storage: network failure, timeout, outage, throttling, an unreachable - /// account, or an authorization failure. All of these are reconfigurable or self-healing, and they are not - /// subdivided because already carries the specific - /// status. Reported with . + /// Reported with . The deletion failed against storage: + /// network failure, timeout, outage, throttling, an unreachable account, or an authorization failure. All of + /// these are reconfigurable or self-healing, and they are not subdivided here because + /// already carries the specific status. Subdividing + /// would encode the same fact twice. /// StorageFailure = 10, /// - /// The registered payload store does not implement deletion. Every payload fails the same way, so this is a - /// deployment-wide condition rather than a per-row one, and it stays recoverable until an operator registers - /// a store that can delete. is empty because storage - /// was never contacted, which is why this is not folded into . Reported with - /// . + /// Reported with . The registered payload store does not + /// implement deletion. Every payload fails the same way, so this is a deployment-wide condition rather than a + /// per-row one, and it is kept recoverable until an operator registers a store that can delete. + /// is empty because storage was never contacted, which + /// is why this cannot be folded into . /// StoreCannotDelete = 11, /// - /// The token cannot be acted on as it stands: its body does not parse, it names a version this worker does - /// not support, or storage rejected it as permanently invalid. + /// The worker could not act on the token. This reason is reported with TWO dispositions, and a consumer must + /// not assume either one. + /// + /// Reported with when the token can never become + /// usable: its body does not parse, it is a legacy v1 token, or storage rejected a well-formed token as + /// permanently invalid. The SDK and backend control both sides of this protocol, so reaching that state + /// indicates a producer, corruption, or compatibility bug, and the evidence is preserved rather than + /// discarded. + /// + /// + /// Reported with in exactly one case: the token names a + /// version this worker does not understand. Nothing is wrong with that token, since a newer worker can read + /// it, so an SDK upgrade resolves it. The asymmetry is deliberate: quarantining it would be permanent and + /// unrecoverable, whereas a retry that never succeeds only leaves the row idle and visible. + /// /// distinguishes the storage-rejected case, where it - /// is populated, from the parse cases, where it is empty. Reported with - /// , except for an unsupported version prefix, which - /// is reported with because an SDK upgrade resolves it. + /// is populated, from the parse and unknown-version cases, where it is empty. /// TokenNotPurgeable = 20, } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 02ec0aee..863f41b0 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -885,49 +885,58 @@ enum LargePayloadPurgeDisposition { // number of distinct causes, because `storageErrorCode` already carries the specific storage status. // Values must never carry a token or raw exception text: a token exposes the storage account, // container, and blob path. +// +// Reason and disposition are orthogonal and there is no fixed mapping between them. A reason says +// what the worker found; a disposition says whether trying again can change it. Most reasons occur +// with exactly one disposition, but TOKEN_NOT_PURGEABLE deliberately occurs with two. Do not assert +// a reason-to-disposition mapping anywhere. enum LargePayloadPurgeReason { LARGE_PAYLOAD_PURGE_REASON_UNSPECIFIED = 0; - // --- Reported with DELETED --- - - // The blob was deleted by this attempt, reclaiming its bytes. + // Reported with DELETED. The blob was deleted by this attempt, reclaiming its bytes. LARGE_PAYLOAD_PURGE_REASON_BLOB_DELETED = 1; - // The blob was already absent, so deletion was a no-op and no bytes were reclaimed by this - // attempt. Deletion is idempotent, so this is a success rather than a failure. Reported - // separately from BLOB_DELETED because a high rate of it indicates duplicate tombstones. + // Reported with DELETED. The blob was already absent, so deletion was a no-op and no bytes were + // reclaimed by this attempt. Deletion is idempotent, so this is a success rather than a failure. + // Reported separately from BLOB_DELETED because a high rate of it indicates duplicate tombstones. LARGE_PAYLOAD_PURGE_REASON_BLOB_ALREADY_ABSENT = 2; - // The blob was left in place because it did not carry the payload store's ownership marker, - // meaning the store did not write it. The token text merely matched the v2 grammar; the payload - // column is customer-writable, so matching text is not proof of ownership. This is an expected - // outcome rather than a defect. The tombstone is still resolved, because a blob the store does - // not own will never become deletable and retrying forever would leak the row. Reported - // separately from BLOB_DELETED so that "resolved without reclaiming bytes" stays countable. + // Reported with DELETED. The blob was left in place because it did not carry the payload store's + // ownership marker, meaning the store did not write it. The token text merely matched the v2 + // grammar; the payload column is customer-writable, so matching text is not proof of ownership. + // This is an expected outcome rather than a defect. The tombstone is still resolved, because a + // blob the store does not own will never become deletable and retrying forever would leak the + // row. Reported separately so that "resolved without reclaiming bytes" stays countable. LARGE_PAYLOAD_PURGE_REASON_BLOB_NOT_STORE_OWNED = 3; - // --- Reported with RETRY --- - - // The deletion failed against storage: network failure, timeout, outage, throttling, an - // unreachable account, or an authorization failure. All of these are reconfigurable or - // self-healing, and they are not subdivided here because `storageErrorCode` already carries the - // specific status for diagnostics. Subdividing would encode the same fact twice. + // Reported with RETRY. The deletion failed against storage: network failure, timeout, outage, + // throttling, an unreachable account, or an authorization failure. All of these are + // reconfigurable or self-healing, and they are not subdivided here because `storageErrorCode` + // already carries the specific status. Subdividing would encode the same fact twice. LARGE_PAYLOAD_PURGE_REASON_STORAGE_FAILURE = 10; - // The registered payload store does not implement deletion. Every payload fails the same way, - // so this is a deployment-wide condition rather than a per-row one, and it is kept recoverable - // until an operator registers a store that can delete. `storageErrorCode` is empty because - // storage was never contacted, which is why this cannot be folded into STORAGE_FAILURE. + // Reported with RETRY. The registered payload store does not implement deletion. Every payload + // fails the same way, so this is a deployment-wide condition rather than a per-row one, and it + // is kept recoverable until an operator registers a store that can delete. `storageErrorCode` is + // empty because storage was never contacted, which is why this cannot be folded into + // STORAGE_FAILURE. LARGE_PAYLOAD_PURGE_REASON_STORE_CANNOT_DELETE = 11; - // --- Reported with QUARANTINED --- - - // The token cannot be acted on and no retry can change that: its body does not parse, it names - // a version this worker does not support, or storage rejected it as permanently invalid. The - // producer and consumer of this token are both controlled by the SDK and backend, so reaching - // this state indicates a producer, corruption, or compatibility bug and the evidence is - // preserved for investigation rather than discarded. `storageErrorCode` distinguishes the - // storage-rejected case, where it is populated, from the parse cases, where it is empty. + // The worker could not act on the token. This reason is reported with TWO dispositions, and a + // consumer must not assume either one. + // + // Reported with QUARANTINED when the token can never become usable: its body does not parse, it + // is a legacy v1 token, or storage rejected a well-formed token as permanently invalid. The SDK + // and backend control both sides of this protocol, so reaching that state indicates a producer, + // corruption, or compatibility bug, and the evidence is preserved rather than discarded. + // + // Reported with RETRY in exactly one case: the token names a version this worker does not + // understand. Nothing is wrong with that token, since a newer worker can read it, so an SDK + // upgrade resolves it. The asymmetry is deliberate: quarantining it would be permanent and + // unrecoverable, whereas a retry that never succeeds only leaves the row idle and visible. + // + // `storageErrorCode` distinguishes the storage-rejected case, where it is populated, from the + // parse and unknown-version cases, where it is empty. LARGE_PAYLOAD_PURGE_REASON_TOKEN_NOT_PURGEABLE = 20; } From e432d2543c09c425121ed43c6ceb6fcddacf9b49 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 16:46:42 -0700 Subject: [PATCH 24/47] Remove purge reason and storage error code from the contract Both fields were verified write-only across the whole system: the backend persists them and nothing reads either one - no SELECT, no WHERE, no API, no alert. Disposition alone drives backend behavior, so the enum was a lossy copy of information that already lives in worker telemetry. Deleting reason also deletes the hazard it created. TOKEN_NOT_PURGEABLE spanned both Retry and Quarantined, so any consumer deriving disposition from reason would have silently stranded rows an SDK upgrade would fix. All 12 activity branches keep their disposition unchanged. Verified branch by branch against the pre-change file rather than by inspection: the unknown-prefix branch stays Retry, and the v1-prefix, malformed-token, and HTTP 400 branches stay Quarantined. Failure detail is now logged rather than sent over the wire. Design section 7 forbids logging raw exceptions, so each classification site logs a bounded cause literal instead. Logging at the classification site rather than at the call site restores the full 11-way granularity the 7-value enum had lost: 401/403, 5xx, and an unreachable account are now distinct in telemetry where the enum had collapsed all three into STORAGE_FAILURE. Two deliberate deviations from the request, both reported: 1. LargePayloadPurgeEnumParityTests.cs is kept, not deleted. It always tested two enums, and the disposition cast in GrpcDurableTaskClient survives - it is the cast that decides whether a row is deleted or quarantined. Only the reason parity fact is removed. 2. This is not the last outbound enum cast. LargePayloadPurgeResult still carries Disposition, so the no-inbound-enum invariant test still guards a live cast and is not vacuous. CA1873 at BlobPurgeJobOrchestrator.cs:66 remains deliberately unfixed: unguarded logging is the established convention here, 77 times solution-wide. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- src/Client/Core/LargePayloadPurgeReason.cs | 90 --------------- src/Client/Core/LargePayloadPurgeResult.cs | 23 ++-- src/Client/Grpc/GrpcDurableTaskClient.cs | 7 +- .../Activities/DeleteExternalBlobActivity.cs | 105 ++++++++---------- .../AzureBlobPayloads/AutoPurge/Logs.cs | 8 +- .../AutoPurge/Models/BlobPurgeOutcome.cs | 15 +-- .../BlobPurgeJobOrchestrator.cs | 4 +- src/Grpc/orchestrator_service.proto | 79 ++----------- .../LargePayloadPurgeEnumParityTests.cs | 27 +---- .../DeleteExternalBlobActivityTests.cs | 98 +++++++++------- 10 files changed, 144 insertions(+), 312 deletions(-) delete mode 100644 src/Client/Core/LargePayloadPurgeReason.cs diff --git a/src/Client/Core/LargePayloadPurgeReason.cs b/src/Client/Core/LargePayloadPurgeReason.cs deleted file mode 100644 index 4984bf35..00000000 --- a/src/Client/Core/LargePayloadPurgeReason.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.Client; - -/// -/// Why a row received its . Diagnostic only: the backend acts on the -/// disposition alone and never branches on this value, so it exists to make a stuck or non-reclaiming ledger -/// explainable without access to worker logs. Deliberately coarse - granularity matches the number of distinct -/// operator responses, not the number of distinct causes, because -/// already carries the specific storage status. Never -/// carries a token or raw exception text, because a token exposes the storage account, container, and blob path. -/// -/// Reason and disposition are orthogonal and there is no fixed mapping between them. A reason says what the -/// worker found; a disposition says whether trying again can change it. Most reasons occur with exactly one -/// disposition, but deliberately occurs with two. Do not assert a -/// reason-to-disposition mapping anywhere. -/// -/// Mirrors the LargePayloadPurgeReason protobuf enum. -/// -public enum LargePayloadPurgeReason -{ - /// - /// No reason was specified. - /// - Unspecified = 0, - - /// - /// Reported with . The blob was deleted by this attempt, - /// reclaiming its bytes. - /// - BlobDeleted = 1, - - /// - /// Reported with . The blob was already absent, so deletion - /// was a no-op and no bytes were reclaimed by this attempt. Deletion is idempotent, so this is a success - /// rather than a failure. Reported separately from because a high rate of it - /// indicates duplicate tombstones. - /// - BlobAlreadyAbsent = 2, - - /// - /// Reported with . The blob was left in place because it - /// did not carry the payload store's ownership marker, meaning the store did not write it. The token text - /// merely matched the v2 grammar; the payload column is customer-writable, so matching text is not proof of - /// ownership. This is an expected outcome rather than a defect. The tombstone is still resolved, because a - /// blob the store does not own will never become deletable and retrying forever would leak the row. Reported - /// separately so that "resolved without reclaiming bytes" stays countable. - /// - BlobNotStoreOwned = 3, - - /// - /// Reported with . The deletion failed against storage: - /// network failure, timeout, outage, throttling, an unreachable account, or an authorization failure. All of - /// these are reconfigurable or self-healing, and they are not subdivided here because - /// already carries the specific status. Subdividing - /// would encode the same fact twice. - /// - StorageFailure = 10, - - /// - /// Reported with . The registered payload store does not - /// implement deletion. Every payload fails the same way, so this is a deployment-wide condition rather than a - /// per-row one, and it is kept recoverable until an operator registers a store that can delete. - /// is empty because storage was never contacted, which - /// is why this cannot be folded into . - /// - StoreCannotDelete = 11, - - /// - /// The worker could not act on the token. This reason is reported with TWO dispositions, and a consumer must - /// not assume either one. - /// - /// Reported with when the token can never become - /// usable: its body does not parse, it is a legacy v1 token, or storage rejected a well-formed token as - /// permanently invalid. The SDK and backend control both sides of this protocol, so reaching that state - /// indicates a producer, corruption, or compatibility bug, and the evidence is preserved rather than - /// discarded. - /// - /// - /// Reported with in exactly one case: the token names a - /// version this worker does not understand. Nothing is wrong with that token, since a newer worker can read - /// it, so an SDK upgrade resolves it. The asymmetry is deliberate: quarantining it would be permanent and - /// unrecoverable, whereas a retry that never succeeds only leaves the row idle and visible. - /// - /// distinguishes the storage-rejected case, where it - /// is populated, from the parse and unknown-version cases, where it is empty. - /// - TokenNotPurgeable = 20, -} diff --git a/src/Client/Core/LargePayloadPurgeResult.cs b/src/Client/Core/LargePayloadPurgeResult.cs index f63f45fb..0a3ee98a 100644 --- a/src/Client/Core/LargePayloadPurgeResult.cs +++ b/src/Client/Core/LargePayloadPurgeResult.cs @@ -6,11 +6,19 @@ namespace Microsoft.DurableTask.Client; /// /// Serializable outcome of exactly one attempted large-payload blob deletion. Mirrors the /// LargePayloadPurgeResult protobuf message but is safe to pass through the orchestration/activity -/// boundary. The backend owns retry scheduling: it deletes rows reported as +/// boundary. The backend owns retry scheduling and branches solely on +/// : it deletes rows reported as /// , reschedules -/// with a reason-appropriate next attempt, and moves -/// rows out of the active fetch. +/// on its own backoff, and moves +/// rows out of the active fetch. The worker never +/// computes a retry delay. /// +/// +/// The disposition is deliberately the only outcome field: anything finer would be write-only on the backend. +/// Why an attempt failed stays in the worker's own telemetry, which holds the cause at full fidelity rather +/// than as a lossy classification, and a row is correlated to it by +/// (, , ). +/// /// The backend partition that owns the tombstoned row. /// The orchestration instance key the payload belonged to. /// The backend identifier of the tombstoned payload row. @@ -19,16 +27,9 @@ namespace Microsoft.DurableTask.Client; /// as a compare-and-swap guard. /// /// The disposition of the deletion attempt. -/// The stable reason code explaining the disposition. -/// -/// An optional bounded, sanitized storage status or error code for diagnostics (for example -/// BlobNotFound or 409). Never contains a token or raw exception text. -/// public sealed record LargePayloadPurgeResult( int PartitionId, long InstanceKey, long PayloadId, long Revision, - LargePayloadPurgeDisposition Disposition, - LargePayloadPurgeReason Reason, - string? StorageErrorCode = null); + LargePayloadPurgeDisposition Disposition); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index adaa2116..0fdbf161 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -677,11 +677,10 @@ public override async Task ReportLargePayloadPurgeResultsAsync( PayloadId = result.PayloadId, Revision = result.Revision, - // The managed enums declare the same numeric values as their protobuf counterparts, so the - // disposition and reason map across by value. + // The managed disposition enum declares the same numeric values as its protobuf counterpart, + // so it maps across by value. This is the only enum on the message and it only travels + // outbound, so the SDK can never receive a value it does not know. Disposition = (P.LargePayloadPurgeDisposition)result.Disposition, - Reason = (P.LargePayloadPurgeReason)result.Reason, - StorageErrorCode = result.StorageErrorCode ?? string.Empty, }); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 7304d4db..4d74b601 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -12,8 +12,8 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Activity that deletes a single externalized payload blob given its token, and classifies the attempt as /// , , or -/// with a stable reason code. Deletion is idempotent, -/// so re-delivered tokens and concurrent workers are safe. +/// . Deletion is idempotent, so re-delivered tokens and +/// concurrent workers are safe. /// /// /// The split between retry and quarantine is whether the failure can self-heal, verified against the @@ -37,9 +37,12 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// failure is returned as a disposition rather than thrown. /// /// -/// Per design §7 no log here carries the token or raw exception text - a token exposes the storage account, -/// container, and blob path. Diagnostics are the stable reason enum plus a bounded, sanitized storage error -/// code; the token itself is preserved on the backend's quarantined row. +/// The reported result carries the disposition alone, so every branch below logs its cause where the cause is +/// still exact, rather than deriving it afterwards from a value that crossed the wire. That log is the only +/// record of why an attempt failed. Per design §7 it still carries neither the token nor raw exception text - +/// a token exposes the storage account, container, and blob path - so the cause is a bounded classification +/// string plus a bounded, sanitized storage error code. The token itself is preserved on the backend's +/// quarantined row. /// /// The payload store used to delete blobs. /// The logger instance. @@ -57,23 +60,7 @@ public override async Task RunAsync(TaskActivityContext contex { Check.NotNullOrEmpty(input, nameof(input)); - BlobPurgeOutcome outcome = await this.DeleteAsync(input); - - switch (outcome.Disposition) - { - case LargePayloadPurgeDisposition.Quarantined: - this.logger.BlobPurgeDeleteQuarantined(outcome.Reason.ToString(), outcome.StorageErrorCode); - break; - case LargePayloadPurgeDisposition.Retry: - this.logger.BlobPurgeDeleteRetryable(outcome.Reason.ToString(), outcome.StorageErrorCode); - break; - case LargePayloadPurgeDisposition.Deleted - when outcome.Reason == LargePayloadPurgeReason.BlobNotStoreOwned: - this.logger.BlobPurgeBlobNotStoreOwned(); - break; - } - - return outcome; + return await this.DeleteAsync(input); } /// @@ -106,94 +93,90 @@ async Task DeleteAsync(string token) // the only durable record of the blob, so the row is quarantined instead - the backend preserves // its token as evidence and stops polling it. The backend excludes v1 at insertion time, so // reaching this branch is an invariant violation rather than an expected path. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.TokenNotPurgeable); + this.logger.BlobPurgeDeleteQuarantined("LegacyV1Token", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); } if (!token.StartsWith(BlobPayloadStore.TokenPrefixV2, StringComparison.Ordinal)) { // An unrecognized prefix is most likely a token written by a newer SDK than this worker runs. That - // recovers after an upgrade, so it earns a deferral rather than quarantine. This branch shares - // TokenNotPurgeable with the quarantined token cases, so its disposition is deliberately stated - // here rather than derived from the reason: folding it in would strand rows an upgrade would fix. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.TokenNotPurgeable); + // recovers after an upgrade, so it earns a deferral rather than quarantine. Quarantine is + // permanent and requires an operator to unwind; a deferral only leaves the row idle and visible, + // so an unrecognized token is deliberately kept on the recoverable side of that asymmetry. + this.logger.BlobPurgeDeleteRetryable("UnsupportedTokenVersion", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } try { PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, CancellationToken.None); - return outcome switch + + // The blob exists but this store never wrote it, so it was left untouched. That is an expected + // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it would + // fill the quarantine set with non-defects. The tombstone is still resolved, because a blob the + // store does not own is not the store's to delete. + if (outcome == PayloadDeleteOutcome.NotStoreOwned) { - PayloadDeleteOutcome.Deleted => new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobDeleted), - PayloadDeleteOutcome.AlreadyAbsent => new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobAlreadyAbsent), + this.logger.BlobPurgeBlobNotStoreOwned(); + } - // The blob exists but this store never wrote it, so it was left untouched. That is an expected - // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it - // would fill the quarantine set with non-defects. The tombstone is still resolved, because a - // blob the store does not own is not the store's to delete. - _ => new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobNotStoreOwned), - }; + // Deleted, AlreadyAbsent, and NotStoreOwned are all terminal successes: none can be improved by + // trying again. + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Deleted); } catch (ArgumentException) { // The prefix gate above proves this is a v2 token, so the only remaining decode failure is a v2 // body that does not parse. The SDK and backend control both sides of the protocol, so that // indicates a producer, corruption, or compatibility bug; retrying can never fix it. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.TokenNotPurgeable); + this.logger.BlobPurgeDeleteQuarantined("MalformedToken", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); } catch (NotSupportedException) { // The registered store does not implement deletion. Every payload would fail the same way, so the // work is kept recoverable until an operator registers a store that can delete. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StoreCannotDelete); + this.logger.BlobPurgeDeleteRetryable("StoreCannotDelete", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } catch (PayloadStorageException) { // The token is well formed but points at a storage account this worker's credential cannot reach // (account-key auth is account-specific). Recoverable after a configuration or credential change, // so it is deferred rather than discarded. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageFailure); + this.logger.BlobPurgeDeleteRetryable("StorageAccountUnreachable", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.BadRequest) { // Storage rejected a request generated from a well-formed token as permanently invalid (for // example InvalidUri / InvalidResourceName). Retrying can never succeed. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, - LargePayloadPurgeReason.TokenNotPurgeable, - SanitizeErrorCode(ex)); + this.logger.BlobPurgeDeleteQuarantined("InvalidStorageRequest", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); } catch (RequestFailedException ex) when ( ex.Status == (int)HttpStatusCode.Unauthorized || ex.Status == (int)HttpStatusCode.Forbidden) { // Authorization can be transient or fixed by reconfiguration, so it stays recoverable rather than // dropping data an operator can still reclaim. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, - LargePayloadPurgeReason.StorageFailure, - SanitizeErrorCode(ex)); + this.logger.BlobPurgeDeleteRetryable("StorageAuthorizationFailed", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } catch (RequestFailedException ex) { // Throttling, 5xx, and anything else the service reported, including a failed If-Match on the // ownership check: transient by default. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, - LargePayloadPurgeReason.StorageFailure, - SanitizeErrorCode(ex)); + this.logger.BlobPurgeDeleteRetryable("TransientStorageFailure", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // Timeouts, cancellation, and network failures. A blob is never dropped on an uncertain error. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageFailure); + // Storage reported no code here, so the exception's type name is appended to the cause: it is a + // bounded value that cannot carry a token, and it is the only thing separating a timeout from a + // cancellation or a DNS failure now that no classification crosses the wire. + this.logger.BlobPurgeDeleteRetryable($"UnexpectedFailure:{ex.GetType().Name}", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index c20cc5d4..8b40c970 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -19,8 +19,8 @@ static partial class Logs [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] public static partial void BlobPurgeJobOrchestratorStopping(this ILogger logger, string? jobId, string status); - [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Blob payload auto-purge quarantined a payload; reason '{reason}', storage error code '{storageErrorCode}'. The failure is deterministic and cannot succeed on a retry. The backend preserves the tombstone row and its token as evidence and stops polling it.")] - public static partial void BlobPurgeDeleteQuarantined(this ILogger logger, string reason, string? storageErrorCode); + [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Blob payload auto-purge quarantined a payload; cause '{cause}', storage code '{storageCode}'. The failure is deterministic and cannot succeed on a retry. The backend preserves the tombstone row and its token as evidence and stops polling it. The reported result carries the disposition alone, so this log is the only record of the cause.")] + public static partial void BlobPurgeDeleteQuarantined(this ILogger logger, string cause, string? storageCode); [LoggerMessage(EventId = 814, Level = LogLevel.Debug, Message = "Blob payload auto-purge fetched {count} tombstoned payload(s) from the backend.")] public static partial void BlobPurgeFetchedTombstones(this ILogger logger, int count); @@ -34,8 +34,8 @@ static partial class Logs [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); - [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; reason '{reason}', storage error code '{storageErrorCode}'. The backend reschedules the tombstone for a later attempt.")] - public static partial void BlobPurgeDeleteRetryable(this ILogger logger, string reason, string? storageErrorCode); + [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; cause '{cause}', storage code '{storageCode}'. The backend reschedules the tombstone for a later attempt. The reported result carries the disposition alone, so this log is the only record of the cause.")] + public static partial void BlobPurgeDeleteRetryable(this ILogger logger, string cause, string? storageCode); [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs index e5c563ec..7f4a1714 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs @@ -10,13 +10,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// orchestrator combines it with the tombstone's identity and revision to build the reported /// . /// +/// +/// Carries the disposition alone, because that is the only outcome field the contract reports. Why an attempt +/// reached its disposition is logged by at the point it is +/// classified, at higher fidelity than any value that could be carried here. +/// /// Whether the row is resolved, should be retried, or must be quarantined. -/// The stable reason code explaining the disposition. -/// -/// An optional bounded, sanitized storage status or error code for diagnostics. Never a token or raw -/// exception text. -/// -public sealed record BlobPurgeOutcome( - LargePayloadPurgeDisposition Disposition, - LargePayloadPurgeReason Reason, - string? StorageErrorCode = null); +public sealed record BlobPurgeOutcome(LargePayloadPurgeDisposition Disposition); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 7d1730c6..4e964612 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -184,8 +184,6 @@ async Task DeleteOneAsync( tombstone.InstanceKey, tombstone.PayloadId, tombstone.Revision, - outcome.Disposition, - outcome.Reason, - outcome.StorageErrorCode); + outcome.Disposition); } } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 863f41b0..51f490a2 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -872,74 +872,14 @@ enum LargePayloadPurgeDisposition { // The failure may self-heal, so the row stays pending and the backend sets the next attempt. LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2; - // A deterministic failure or protocol violation that retrying can never fix. The backend - // preserves the evidence, alerts, and stops automatic retries. + // A deterministic failure or protocol violation that retrying can never fix. The row leaves the + // polling set but is never deleted or expired: it keeps the token, which after the payload row is + // gone is the only durable record of the blob, so discarding it would orphan the blob silently. + // Resolving a quarantined row is a deliberate operator action. Why it failed is not recorded here + // and is not meant to be; that detail lives in the worker's telemetry at full fidelity. LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; } -// Why a row received its disposition. Diagnostic only: the backend acts on `disposition` alone and -// never branches on this value, so it exists to make a stuck or non-reclaiming ledger explainable -// without access to worker logs, which run in the customer's process. -// -// Deliberately coarse. Granularity matches the number of distinct operator responses, not the -// number of distinct causes, because `storageErrorCode` already carries the specific storage status. -// Values must never carry a token or raw exception text: a token exposes the storage account, -// container, and blob path. -// -// Reason and disposition are orthogonal and there is no fixed mapping between them. A reason says -// what the worker found; a disposition says whether trying again can change it. Most reasons occur -// with exactly one disposition, but TOKEN_NOT_PURGEABLE deliberately occurs with two. Do not assert -// a reason-to-disposition mapping anywhere. -enum LargePayloadPurgeReason { - LARGE_PAYLOAD_PURGE_REASON_UNSPECIFIED = 0; - - // Reported with DELETED. The blob was deleted by this attempt, reclaiming its bytes. - LARGE_PAYLOAD_PURGE_REASON_BLOB_DELETED = 1; - - // Reported with DELETED. The blob was already absent, so deletion was a no-op and no bytes were - // reclaimed by this attempt. Deletion is idempotent, so this is a success rather than a failure. - // Reported separately from BLOB_DELETED because a high rate of it indicates duplicate tombstones. - LARGE_PAYLOAD_PURGE_REASON_BLOB_ALREADY_ABSENT = 2; - - // Reported with DELETED. The blob was left in place because it did not carry the payload store's - // ownership marker, meaning the store did not write it. The token text merely matched the v2 - // grammar; the payload column is customer-writable, so matching text is not proof of ownership. - // This is an expected outcome rather than a defect. The tombstone is still resolved, because a - // blob the store does not own will never become deletable and retrying forever would leak the - // row. Reported separately so that "resolved without reclaiming bytes" stays countable. - LARGE_PAYLOAD_PURGE_REASON_BLOB_NOT_STORE_OWNED = 3; - - // Reported with RETRY. The deletion failed against storage: network failure, timeout, outage, - // throttling, an unreachable account, or an authorization failure. All of these are - // reconfigurable or self-healing, and they are not subdivided here because `storageErrorCode` - // already carries the specific status. Subdividing would encode the same fact twice. - LARGE_PAYLOAD_PURGE_REASON_STORAGE_FAILURE = 10; - - // Reported with RETRY. The registered payload store does not implement deletion. Every payload - // fails the same way, so this is a deployment-wide condition rather than a per-row one, and it - // is kept recoverable until an operator registers a store that can delete. `storageErrorCode` is - // empty because storage was never contacted, which is why this cannot be folded into - // STORAGE_FAILURE. - LARGE_PAYLOAD_PURGE_REASON_STORE_CANNOT_DELETE = 11; - - // The worker could not act on the token. This reason is reported with TWO dispositions, and a - // consumer must not assume either one. - // - // Reported with QUARANTINED when the token can never become usable: its body does not parse, it - // is a legacy v1 token, or storage rejected a well-formed token as permanently invalid. The SDK - // and backend control both sides of this protocol, so reaching that state indicates a producer, - // corruption, or compatibility bug, and the evidence is preserved rather than discarded. - // - // Reported with RETRY in exactly one case: the token names a version this worker does not - // understand. Nothing is wrong with that token, since a newer worker can read it, so an SDK - // upgrade resolves it. The asymmetry is deliberate: quarantining it would be permanent and - // unrecoverable, whereas a retry that never succeeds only leaves the row idle and visible. - // - // `storageErrorCode` distinguishes the storage-rejected case, where it is populated, from the - // parse and unknown-version cases, where it is empty. - LARGE_PAYLOAD_PURGE_REASON_TOKEN_NOT_PURGEABLE = 20; -} - // client -> server: the outcome of exactly one tombstoned row. message LargePayloadPurgeResult { // Row identity, echoed from the corresponding LargePayloadTombstone. @@ -950,12 +890,11 @@ message LargePayloadPurgeResult { // Echoed unmodified from the fetched tombstone; used as a compare-and-swap guard. int64 revision = 4; + // The only field the backend acts on. Deliberately the only outcome field on this message: + // anything finer would be write-only. Failure detail stays in the worker's own telemetry, which + // holds the full exception rather than a lossy classification, and a row is correlated to it by + // (partitionId, instanceKey, payloadId) plus the ledger's LastAttemptAt. LargePayloadPurgeDisposition disposition = 5; - LargePayloadPurgeReason reason = 6; - - // Optional bounded, sanitized storage status or error code for diagnostics - // (for example "BlobNotFound" or "409"). Must never contain a token or raw exception text. - string storageErrorCode = 7; } // client -> server: request up to `limit` due tombstones for the caller's task hub. diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs index bd563897..48658410 100644 --- a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs +++ b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs @@ -8,9 +8,9 @@ namespace Microsoft.DurableTask.Client.Grpc.Tests; /// -/// maps the managed purge enums onto -/// their protobuf counterparts by numeric value rather than by name, which is only correct while the two sides -/// agree on every value. A silent drift would not fail to compile; it would send the backend a different +/// maps the managed purge disposition +/// onto its protobuf counterpart by numeric value rather than by name, which is only correct while the two +/// sides agree on every value. A silent drift would not fail to compile; it would send the backend a different /// disposition than the worker decided and delete or quarantine the wrong rows. These tests pin the mapping. /// public class LargePayloadPurgeEnumParityTests @@ -31,24 +31,9 @@ public void Disposition_ManagedAndProtobufValues_AreIdentical() managed.Should().Equal(proto); } - [Fact] - public void Reason_ManagedAndProtobufValues_AreIdentical() - { - // Arrange & Act - Dictionary managed = Enum.GetValues(typeof(LargePayloadPurgeReason)) - .Cast() - .ToDictionary(v => (int)v, v => v.ToString()); - Dictionary proto = Enum.GetValues(typeof(P.LargePayloadPurgeReason)) - .Cast() - .ToDictionary(v => (int)v, v => v.ToString()); - - // Assert - managed.Should().Equal(proto); - } - /// - /// The numeric casts are safe only because no enum crosses the wire inbound on this feature: the SDK - /// casts values it defined itself, so it can never receive an unknown value and silently reinterpret it. + /// The numeric cast is safe only because no enum crosses the wire inbound on this feature: the SDK + /// casts a value it defined itself, so it can never receive an unknown value and silently reinterpret it. /// That invariant holds today by the shape of the contract, not by construction, and nothing in the code /// states it. Adding an enum to an inbound type would create exactly that path - a newer backend sending a /// value this SDK does not know, mapped by raw numeric cast onto a valid-but-wrong member - and it would @@ -71,7 +56,7 @@ public void InboundTypes_ExposeNoEnumMembers(Type inboundType) // Assert enumMembers.Should().BeEmpty( - "an enum on an inbound type invalidates the numeric enum casts in " + + "an enum on an inbound type invalidates the numeric enum cast in " + "GrpcDurableTaskClient.ReportLargePayloadPurgeResultsAsync. The SDK would map a value chosen by the " + "backend - including one a newer backend added that this SDK does not know - onto a managed member " + "by raw numeric value, silently mis-dispositioning rows. Map inbound enums explicitly instead, with " + diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 7949cc57..94ad4830 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -9,25 +9,32 @@ namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; +/// +/// The reported outcome carries the disposition alone, so each case pins two things: the disposition, which is +/// what the backend acts on, and the logged cause, which is now the only record of why an attempt reached that +/// disposition. Several branches share a disposition and are told apart only by their cause, so asserting the +/// disposition alone would let two branches collapse into one unnoticed. +/// public class DeleteExternalBlobActivityTests { const string V2Token = "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"; [Fact] - public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesAsTokenNotPurgeable() + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_Quarantines() { // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. StubPayloadStore store = new(new RequestFailedException(400, "bad", "InvalidResourceName", null)); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - quarantined (evidence preserved), never a success-shaped discard. The storage error code - // is what distinguishes this from the parse-failure cases, which share the same reason. + // Assert - quarantined (evidence preserved), never a success-shaped discard. The sanitized storage + // error code is logged alongside the cause and is what distinguishes this from the parse failures. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); - outcome.StorageErrorCode.Should().Be("InvalidResourceName"); + logger.Logs.Should().ContainSingle( + l => l.Message.Contains("InvalidStorageRequest") && l.Message.Contains("InvalidResourceName")); } [Fact] @@ -35,15 +42,16 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_RetriesAsTransien { // Arrange - a Status 503 that escaped the SDK's internal retries is still treated as transient. StubPayloadStore store = new(new RequestFailedException(503, "busy", "ServerBusy", null)); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); - outcome.StorageErrorCode.Should().Be("ServerBusy"); + logger.Logs.Should().ContainSingle( + l => l.Message.Contains("TransientStorageFailure") && l.Message.Contains("ServerBusy")); } [Theory] @@ -53,31 +61,34 @@ public async Task RunAsync_WhenDeleteThrowsAuthorizationFailure_Retries(int stat { // Arrange - authorization can be fixed by reconfiguration, so it stays recoverable. StubPayloadStore store = new(new RequestFailedException(status, "denied", "AuthorizationFailure", null)); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StorageAuthorizationFailed")); } [Fact] - public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsStorageFailure() + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_Retries() { // Arrange - the payload lives in a storage account the configured credential cannot reach. That is // recoverable after a configuration or credential change, so it is deferred rather than discarded. StubPayloadStore store = new(new PayloadStorageException("cross-account delete requires identity auth")); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync( null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); - // Assert + // Assert - the cause is what separates this from the other retryable storage failures, which the + // contract no longer distinguishes. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StorageAccountUnreachable")); } [Fact] @@ -86,14 +97,15 @@ public async Task RunAsync_V1Token_QuarantinesWithoutCallingStore() // Arrange - a v1 token names a container but not the storage account, so a delete against the // configured account cannot be verified and would falsely report success if the store was repointed. Mock store = new(); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); // Assert - quarantined by the gate, and the store's DeleteAsync was never invoked. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("LegacyV1Token")); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -102,16 +114,18 @@ public async Task RunAsync_UnknownTokenVersion_RetriesWithoutCallingStore() { // Arrange - an unrecognized prefix most likely came from a newer SDK, which recovers after an upgrade. Mock store = new(); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v9:https://acct.blob.core.windows.net/c/x"); - // Assert - retried, NOT quarantined: quarantine is terminal and this can self-heal. This is the one - // row where TokenNotPurgeable does not mean Quarantined, so the disposition is asserted deliberately: - // deriving it from the reason would strand rows that an SDK upgrade would have resolved. + // Assert - retried, NOT quarantined: quarantine is terminal and requires an operator to unwind, while a + // deferral only leaves the row idle and visible. This branch and the token branches that quarantine are + // told apart only by cause, so the disposition is asserted deliberately: folding them together would + // strand rows that an SDK upgrade would have resolved. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnsupportedTokenVersion")); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -121,14 +135,15 @@ public async Task RunAsync_MalformedV2Token_Quarantines() // Arrange - a recognized v2 prefix whose body does not parse; the store signals that with // ArgumentException. Retrying can never fix a body the SDK itself produced malformed. StubPayloadStore store = new(new ArgumentException("Invalid token")); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v2:not-a-uri"); - // Assert - contrast with the unknown-prefix case above, which shares this reason but is retried. + // Assert - contrast with the unknown-prefix case above, which is retried. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("MalformedToken")); } [Fact] @@ -138,14 +153,15 @@ public async Task RunAsync_V2Token_CallsStoreAndReportsDeleted() Mock store = new(); store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(PayloadDeleteOutcome.Deleted); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert + // Assert - an ordinary success is silent; any log here would mean a failure branch was taken. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); - outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobDeleted); + logger.Logs.Should().BeEmpty(); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Once); } @@ -156,14 +172,15 @@ public async Task RunAsync_WhenBlobAlreadyAbsent_ReportsDeleted() Mock store = new(); store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(PayloadDeleteOutcome.AlreadyAbsent); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); - outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobAlreadyAbsent); + logger.Logs.Should().BeEmpty(); } [Fact] @@ -173,15 +190,17 @@ public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeletin Mock store = new(); store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(PayloadDeleteOutcome.NotStoreOwned); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); // Assert - the tombstone is still resolved: a blob the store does not own is not the store's to delete, - // and re-serving the row forever would never make it deletable. + // and re-serving the row forever would never make it deletable. The reported result is now identical to + // an ordinary delete, so the log is the only thing that records the difference. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); - outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobNotStoreOwned); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("ownership marker")); } [Fact] @@ -189,7 +208,8 @@ public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombst { // Arrange - a store that cannot delete (the base PayloadStore.DeleteAsync throws NotSupportedException). StubPayloadStore store = new(new NotSupportedException()); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); @@ -197,7 +217,7 @@ public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombst // Assert - retried (tombstone preserved): resolving it would destroy the backend's cleanup ledger while // the blob survives. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StoreCannotDelete); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StoreCannotDelete")); } [Fact] @@ -205,15 +225,15 @@ public async Task RunAsync_WhenDeleteTimesOut_RetriesAsTransient() { // Arrange - a non-Azure exception (timeout / network failure) must not drop a blob on doubt. StubPayloadStore store = new(new TimeoutException()); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert + // Assert - storage reported no code here, so the exception's type name is what identifies the failure. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); - outcome.StorageErrorCode.Should().BeNull(); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnexpectedFailure:TimeoutException")); } sealed class StubPayloadStore : PayloadStore From 1ac131ff00a53e66b601251a3762adbf84acc4e5 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 11:35:54 -0700 Subject: [PATCH 25/47] Dedupe the purge bridge in the backend instead of checking first EnsureJobAsync read the bridge orchestration's status and then scheduled it. Those are two independent RPCs with no atomicity between them, so two hosts starting together could both observe "absent" and both schedule. The status check is replaced by a dedupe policy on the create call, which makes the decision atomic in the backend. The policy is the exact semantic equivalent of the predicate it replaces: the old code rescheduled only when the bridge was absent, Failed, or Terminated, so Failed and Terminated are the only replaceable statuses and every other status is deduped. The list must name all five explicitly because the wire format is an inverted whitelist - the policy is computed as (all statuses - dedupe statuses), so any status omitted silently becomes replaceable. Pending in particular is not optional: instances are created Pending and only become Running after awaiting their first task, so leaving it out would open a real window in which a just-scheduled bridge is replaced. This also makes the OrchestrationAlreadyExistsException handler live. It was previously dead code: the exception is only thrown for statuses in DedupeStatuses, and no policy was supplied. It is now the normal steady-state outcome on every host start after the first, so its comment is corrected rather than its behavior. OrchestrationRuntimeStatus.Canceled is [Obsolete], so naming it emits CS0618. It is suppressed with a scoped pragma, matching how ProtoUtils and StartOrchestrationOptionsExtensions handle the same member. Dropping the status instead would have silently changed behavior on Canceled. Adds a test pinning the exact dedupe set. An omitted status is invisible to the compiler and would surface only as a healthy singleton being purged and replaced in production; EnsureJobAsync had no coverage before this. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 60 ++++++++++++------- .../AutoPurge/BlobPurgeJobStarterTests.cs | 46 ++++++++++++++ 2 files changed, 84 insertions(+), 22 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 3508120d..a2825430 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -129,41 +129,57 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT { try { - // The singleton is already guaranteed by the entity's fixed key (Create no-ops when the job is - // active) and the orchestrator's fixed instance id. The bridge orchestration's only job is to - // apply the entity's Create once, under a fixed instance id. Before (re)scheduling it, check the - // existing bridge: if it already Completed - or is still alive (Running/Pending/Suspended) - the - // job is set up, so do not reschedule. (Re-running a Completed bridge is wasteful: with a fixed - // id and no dedupe policy the backend would purge and replace the terminal instance on every - // host restart.) Only (re)schedule when the bridge is absent, or ended in a Failed/Terminated - // state that may never have applied Create - which lets a failed setup self-heal. - OrchestrationMetadata? existing = await client.GetInstanceAsync( - BlobPurgeConstants.StarterInstanceId, cancellationToken); - - bool needsSchedule = existing is null - or { RuntimeStatus: OrchestrationRuntimeStatus.Failed or OrchestrationRuntimeStatus.Terminated }; - if (!needsSchedule) - { - this.logger.BlobPurgeJobEnsured(); - return; - } - BlobPurgeJobOperationRequest request = new( this.entityId, nameof(BlobPurgeJob.Create), batchSize); + // The bridge orchestration's only job is to apply the entity's Create once, under a fixed + // instance id. Whether it should be (re)scheduled is decided by the backend as part of the + // create call, not by reading its status first: a read-then-schedule pair is two independent + // RPCs with no atomicity between them, so two hosts starting together can both observe "absent" + // and both schedule. + // + // The dedupe list is an inverted whitelist. The wire policy is computed as + // (all statuses - dedupe statuses) = the statuses that may be REPLACED, so any status left out + // of this list silently becomes replaceable. It must therefore name every status that must not + // be disturbed, not just the interesting ones: + // Completed - re-running a finished bridge is wasteful, and with a fixed id the backend + // would purge and replace the terminal instance on every host restart. There is + // nothing to gain: the bridge would only re-signal Create, which no-ops while + // the entity is Active. + // Canceled - terminal and not a failed setup, so it is left alone like Completed. + // Pending - instances are created Pending and only become Running after awaiting their + // first task, so omitting it would leave a real window in which a just-scheduled + // bridge is replaced. + // Running, + // Suspended - still alive; the job is already set up. + // Failed and Terminated are deliberately absent, which makes them the only replaceable + // statuses: those may never have applied Create, so rescheduling is what lets a failed setup + // self-heal. Terminated must also stay out of the list because supplying it alongside a + // reusable running status is rejected outright. +#pragma warning disable CS0618 // Canceled is obsolete, but is a real dedupe status and must be named here. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, - new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId), + new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId) + .WithDedupeStatuses( + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Canceled, + OrchestrationRuntimeStatus.Pending, + OrchestrationRuntimeStatus.Running, + OrchestrationRuntimeStatus.Suspended), cancellationToken); +#pragma warning restore CS0618 this.logger.BlobPurgeJobEnsured(); return; } catch (OrchestrationAlreadyExistsException) { - // Race: another client scheduled the bridge between our status check and schedule call. That is - // fine - the singleton is already kicked off; treat it as ensured and stop. + // The expected steady-state outcome, not a rare race: the backend throws this whenever the + // bridge already exists in one of the dedupe statuses above, which is every host start after + // the first one succeeded. It means the singleton is already set up, so treat it as ensured + // and stop. It also covers the race this replaced a status check to close - two hosts starting + // together - because exactly one create wins and the loser lands here. this.logger.BlobPurgeJobEnsured(); return; } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index f79c79e6..a5c79d32 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -86,6 +86,52 @@ public async Task StartAsync_WhenAutoPurgeDisabled_DoesNotResolveClientOrLog() logger.Logs.Should().BeEmpty(); } + [Fact] + public async Task EnsureJob_SchedulesBridge_DedupingEveryStatusExceptFailedAndTerminated() + { + // Arrange - the dedupe list is an inverted whitelist: the wire policy is (all statuses - dedupe), so a + // status omitted from the call silently becomes replaceable. Nothing in the compiler or the type system + // catches that, so the exact set is pinned here. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + Mock client = new("test"); + TaskCompletionSource scheduled = new(); + client + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback( + (_, _, options, _) => scheduled.TrySetResult(options)) + .ReturnsAsync(BlobPurgeConstants.StarterInstanceId); + + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(client.Object); + BlobPurgeJobStarter starter = new( + provider.Object, + store, + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = true }), + "test", + new TestLogger()); + + // Act - the ensure work runs on a background task, so wait for the scheduling call rather than assuming + // it already happened. + await starter.StartAsync(CancellationToken.None); + Task completed = await Task.WhenAny(scheduled.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await starter.StopAsync(CancellationToken.None); + + // Assert + completed.Should().BeSameAs(scheduled.Task, "the starter must schedule the bridge orchestration"); + StartOrchestrationOptions? options = await scheduled.Task; + options.Should().NotBeNull(); + options!.InstanceId.Should().Be(BlobPurgeConstants.StarterInstanceId); + + // Failed and Terminated are the only replaceable statuses, which is what lets a failed setup self-heal. + // Every other status is deduped so a healthy or finished bridge is never purged and replaced. + options.DedupeStatuses.Should().BeEquivalentTo( + ["Completed", "Canceled", "Pending", "Running", "Suspended"]); + } + static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) { Mock> monitor = new(); From ea0b4c773f6fe7490cf4a17c3fab690d8bbc94a3 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 14:35:49 -0700 Subject: [PATCH 26/47] Dedupe the purge bridge on Pending and Running only The bridge deduped five statuses, which preserved the exact semantics of the status check it replaced. That preserved a dead end along with them: a bridge that had already run was Completed and therefore never re-ran, so there was no path back from a job that had stopped. Deduping only Pending and Running says something narrower - while the bridge is alive leave it alone, in any other state re-run it - and that is what makes the job self-healing. The perpetual orchestrator exits cleanly when it reads back a null entity state, which is reachable when the entity is removed by CleanEntityStorageAsync. A removed entity is back to its default Pending status, so the job is left dead with a Completed bridge behind it. Making Completed replaceable means the next host start re-runs Create, which finds the entity not Active and rebuilds the job. Re-running a finished bridge is cheap and safe: its only effect is calling Create, which no-ops while the entity is Active, so the steady-state cost is one instance replacement plus one entity call per host start. Pending stays in the list. Instances are created Pending and only become Running after awaiting their first task, so omitting it would leave a real window in which a just-scheduled bridge is replaced. Two comments were wrong and are corrected rather than carried forward. The first claimed the backend would purge and replace a terminal instance on every host restart; a null DedupeStatuses field means the backend default, which the contract explicitly leaves undefined, so that asserted behavior the SDK does not promise. The second described the OrchestrationAlreadyExistsException handler as the steady-state outcome. That was true when Completed was deduped, but under this policy a finished bridge is replaced rather than rejected, so the handler is once again the concurrent-start race path it was originally written for. The CS0618 suppression is removed with Canceled. ContinuedAsNew and Canceled are the only obsolete members of OrchestrationRuntimeStatus, so with Canceled gone the pragma covered nothing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 57 ++++++++++--------- .../AutoPurge/BlobPurgeJobStarterTests.cs | 10 ++-- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index a2825430..7a911feb 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -140,46 +140,49 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // // The dedupe list is an inverted whitelist. The wire policy is computed as // (all statuses - dedupe statuses) = the statuses that may be REPLACED, so any status left out - // of this list silently becomes replaceable. It must therefore name every status that must not - // be disturbed, not just the interesting ones: - // Completed - re-running a finished bridge is wasteful, and with a fixed id the backend - // would purge and replace the terminal instance on every host restart. There is - // nothing to gain: the bridge would only re-signal Create, which no-ops while - // the entity is Active. - // Canceled - terminal and not a failed setup, so it is left alone like Completed. - // Pending - instances are created Pending and only become Running after awaiting their - // first task, so omitting it would leave a real window in which a just-scheduled - // bridge is replaced. - // Running, - // Suspended - still alive; the job is already set up. - // Failed and Terminated are deliberately absent, which makes them the only replaceable - // statuses: those may never have applied Create, so rescheduling is what lets a failed setup - // self-heal. Terminated must also stay out of the list because supplying it alongside a - // reusable running status is rejected outright. -#pragma warning disable CS0618 // Canceled is obsolete, but is a real dedupe status and must be named here. + // of this list silently becomes replaceable. Deduping exactly Pending and Running therefore + // means: while the bridge is alive leave it alone, and in any other state re-run it. + // + // Pending is the subtle one and is not optional. Instances are created Pending and only become + // Running after awaiting their first task, so omitting it would leave a real window in which a + // just-scheduled bridge is replaced. + // + // Re-running a bridge that already finished is safe and close to free: the bridge's only effect + // is calling Create, which no-ops while the entity is Active, so the cost is one instance + // replacement plus one entity call per host start. + // + // Re-running is also what lets the job self-heal after the entity is removed, for example by + // CleanEntityStorageAsync. The perpetual orchestrator exits cleanly when it reads back a null + // entity state, and a removed entity is back to its default Pending status, so the job is left + // dead with a Completed bridge behind it. Keeping Completed deduped would keep it dead + // permanently; making it replaceable means the next host start re-runs Create, which finds the + // entity not Active and rebuilds the job. + // + // This deliberately does NOT recover the case where the perpetual orchestrator dies while the + // entity is still Active: the bridge re-runs, Create no-ops on the Active state, and the + // orchestrator stays down. That gap is tracked separately as an open lifecycle item. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId) .WithDedupeStatuses( - OrchestrationRuntimeStatus.Completed, - OrchestrationRuntimeStatus.Canceled, OrchestrationRuntimeStatus.Pending, - OrchestrationRuntimeStatus.Running, - OrchestrationRuntimeStatus.Suspended), + OrchestrationRuntimeStatus.Running), cancellationToken); -#pragma warning restore CS0618 this.logger.BlobPurgeJobEnsured(); return; } catch (OrchestrationAlreadyExistsException) { - // The expected steady-state outcome, not a rare race: the backend throws this whenever the - // bridge already exists in one of the dedupe statuses above, which is every host start after - // the first one succeeded. It means the singleton is already set up, so treat it as ensured - // and stop. It also covers the race this replaced a status check to close - two hosts starting - // together - because exactly one create wins and the loser lands here. + // Thrown only when the bridge already exists in one of the dedupe statuses above, so under this + // policy it means another host scheduled the bridge and it is still Pending or Running. That is + // exactly the concurrent-start race this replaced a status check to close: one create wins and + // the loser lands here. Either way the singleton is already being set up, so treat it as + // ensured and stop. + // + // Note this is NOT the steady-state path. A bridge that already finished is Completed, which is + // replaceable, so a later host start re-runs it rather than landing here. this.logger.BlobPurgeJobEnsured(); return; } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index a5c79d32..2aea6a84 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -87,7 +87,7 @@ public async Task StartAsync_WhenAutoPurgeDisabled_DoesNotResolveClientOrLog() } [Fact] - public async Task EnsureJob_SchedulesBridge_DedupingEveryStatusExceptFailedAndTerminated() + public async Task EnsureJob_SchedulesBridge_DedupingOnlyPendingAndRunning() { // Arrange - the dedupe list is an inverted whitelist: the wire policy is (all statuses - dedupe), so a // status omitted from the call silently becomes replaceable. Nothing in the compiler or the type system @@ -126,10 +126,10 @@ public async Task EnsureJob_SchedulesBridge_DedupingEveryStatusExceptFailedAndTe options.Should().NotBeNull(); options!.InstanceId.Should().Be(BlobPurgeConstants.StarterInstanceId); - // Failed and Terminated are the only replaceable statuses, which is what lets a failed setup self-heal. - // Every other status is deduped so a healthy or finished bridge is never purged and replaced. - options.DedupeStatuses.Should().BeEquivalentTo( - ["Completed", "Canceled", "Pending", "Running", "Suspended"]); + // Every status other than Pending and Running is replaceable, so a finished bridge is re-run on the + // next host start. That is what lets the job rebuild itself after the entity is removed. The set is + // asserted exactly, never as a superset, because the hazard is a silent omission. + options.DedupeStatuses.Should().BeEquivalentTo(["Pending", "Running"]); } static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) From 3c5c7ba0152fdc357496fd27f9e781c716e7c404 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 16:13:16 -0700 Subject: [PATCH 27/47] Add Stop operation and make the purge batch size configurable at runtime Answers the lifecycle gap on the singleton purge job: disabling auto-purge or changing the batch size was permanently ignored once the job existed. Stop: a new entity operation moves the job off Active and is idempotent. The perpetual orchestrator is deliberately not terminated - it already reads the entity at the top of every cycle and already has a clean-exit path, so Stop simply makes that path reachable. Shutdown is cooperative and race-free: the in-flight cycle finishes and the orchestrator exits on its own terms. Disable path: the starter used to return silently when auto-purge was off, so no host ever told the backend and a job created while the flag was on kept deleting blobs forever. It now signals Stop on the same fire-and-forget, retrying background task as the enable path. The store gate is deliberately not applied there: stopping needs no ability to delete, and a user who has switched to a store that cannot delete is exactly who needs the job stopped. Batch size: two independent breaks, both fixed. Create now takes the batch size even when the job is already Active (without re-signalling Run), and the orchestrator reads it from the entity state it already fetches each cycle instead of from its fixed input, falling back to the input when the stored value is not positive. Automatic restart of a dead orchestrator is a deliberate non-goal, documented in place: the only available restart schedules on the entity path, which carries no dedupe policy, so signalling a healthy orchestrator would terminate and replace it mid-work. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 94 ++++++++++++--- .../AutoPurge/Entity/BlobPurgeJob.cs | 56 ++++++++- .../AzureBlobPayloads/AutoPurge/Logs.cs | 11 +- .../AutoPurge/Models/BlobPurgeJobStatus.cs | 6 +- .../BlobPurgeJobOrchestrator.cs | 24 +++- .../BlobPurgeJobOrchestratorTests.cs | 113 ++++++++++++++++++ .../AutoPurge/BlobPurgeJobStarterTests.cs | 81 ++++++++++--- .../AutoPurge/BlobPurgeJobTests.cs | 93 +++++++++++++- 8 files changed, 434 insertions(+), 44 deletions(-) create mode 100644 test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 7a911feb..477eac9b 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -11,12 +11,12 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Client-side hosted service that ensures the per-task-hub singleton blob payload auto-purge job exists. It is -/// registered unconditionally by UseExternalizedPayloads and decides what to do at startup, once options are -/// fully resolved: it no-ops silently when auto-purge is disabled, and no-ops with an error log when the -/// registered store cannot delete. It never blocks host startup - the ensure work runs on a background task -/// that retries until the backend is reachable. The job is a per-task-hub singleton, so racing client -/// processes simply no-op. +/// Client-side hosted service that reconciles the per-task-hub singleton blob payload auto-purge job with the +/// resolved options. It is registered unconditionally by UseExternalizedPayloads and decides what to do at +/// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, asks the job +/// to stop when it is disabled, and no-ops with an error log when the registered store cannot delete. It never +/// blocks host startup - the work runs on a background task that retries until the backend is reachable. The +/// job is a per-task-hub singleton, so racing client processes converge on the same result. /// sealed class BlobPurgeJobStarter : IHostedService, IDisposable { @@ -30,7 +30,7 @@ sealed class BlobPurgeJobStarter : IHostedService, IDisposable readonly EntityInstanceId entityId = new(nameof(BlobPurgeJob), BlobPurgeConstants.JobId); CancellationTokenSource? cts; - Task? ensureTask; + Task? backgroundTask; /// /// Initializes a new instance of the class. @@ -63,10 +63,23 @@ public Task StartAsync(CancellationToken cancellationToken) // auto-purge is enabled can only be known once options are fully resolved - the flag can be set by the // inline configure delegate, services.Configure, configuration binding or PostConfigure. Deciding at // registration time (by running the delegate against a probe instance) both invoked user code twice and - // missed every enable path except the inline delegate. This is the normal path for apps that externalize - // payloads without auto-purge, so it returns silently without logging. + // missed every enable path except the inline delegate. if (!opts.AutoPurge) { + // Turning auto-purge off has to reach the backend to mean anything. The job is a perpetual + // orchestrator owned by the task hub, not by this process, so a job created while the flag was on + // keeps deleting blobs forever no matter how many hosts start with it off. Ask it to stop. + // + // The store gate below deliberately does NOT apply to this path. Stopping a job requires no ability + // to delete anything, and a user who has switched to a store that cannot delete is precisely + // someone who needs the job stopped; that gate's error is worded for the enabled case and would be + // a false alarm here. + // + // One deliberate cost: entity state is persisted after every operation, so this signal creates the + // job entity in its default, not-running state for apps that never enabled auto-purge. That is a + // single small entity per task hub, and it is the same one the job would use if it were enabled. + this.cts = new CancellationTokenSource(); + this.backgroundTask = Task.Run(() => this.SignalJobStopAsync(this.cts.Token), CancellationToken.None); return Task.CompletedTask; } @@ -91,7 +104,7 @@ public Task StartAsync(CancellationToken cancellationToken) // Do not block host startup; ensure the job on a background task with basic retry until the backend // is reachable. this.cts = new CancellationTokenSource(); - this.ensureTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); + this.backgroundTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); return Task.CompletedTask; } @@ -100,20 +113,21 @@ public async Task StopAsync(CancellationToken cancellationToken) { this.cts?.Cancel(); - Task? pending = this.ensureTask; + Task? pending = this.backgroundTask; if (pending is not null) { - // The ensure loop observes cancellation and returns promptly; swallow any faulted/cancelled result. + // The background loop observes cancellation and returns promptly; swallow any faulted/cancelled + // result. await Task.WhenAny(pending, Task.Delay(Timeout.Infinite, cancellationToken)).ConfigureAwait(false); } } /// - /// Disposes the cancellation source backing the background ensure task. + /// Disposes the cancellation source backing the background task. /// /// /// Deliberately not disposed in : that method stops waiting as soon as the host's - /// shutdown token fires, so the ensure task may still hold this source's token. Disposing it there would + /// shutdown token fires, so the background task may still hold this source's token. Disposing it there would /// fault that still-running task with an when it next registers a /// callback. The container disposes singletons after every has returned, which is /// the safe point. @@ -160,7 +174,12 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // // This deliberately does NOT recover the case where the perpetual orchestrator dies while the // entity is still Active: the bridge re-runs, Create no-ops on the Active state, and the - // orchestrator stays down. That gap is tracked separately as an open lifecycle item. + // orchestrator stays down. That is a deliberate non-goal rather than an oversight. The only + // available restart is signalling the entity's Run, which schedules the orchestrator on the + // entity path with no dedupe policy available to it, so signalling a healthy orchestrator would + // terminate and replace it mid-work. Deciding it is safe to signal requires a check-then-act on + // the orchestrator's status, which is exactly the non-atomic pattern this call replaced. + // Recovery is therefore a supervised manual step. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, @@ -204,4 +223,49 @@ await client.ScheduleNewOrchestrationInstanceAsync( } } } + + async Task SignalJobStopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + // Resolved inside the loop rather than on the host-start path. This path is taken by every app + // that externalizes payloads without auto-purge, and constructing a DurableTaskClient at host + // start for them is exactly what the enable path's lazy resolution avoids. It also means a + // client that cannot be resolved yet is retried here instead of throwing out of host startup, + // which must never happen for a feature the user has turned off. + DurableTaskClient client = this.clientProvider.GetClient(this.builderName); + + // Signalled rather than driven through the bridge orchestration: unlike Create, Stop needs no + // fixed-instance dedupe. It is idempotent in the entity, so concurrent hosts converge on the + // same state and a repeat costs nothing beyond the signal itself. + await client.Entities.SignalEntityAsync( + this.entityId, + nameof(BlobPurgeJob.Stop), + null, + null, + cancellationToken); + + this.logger.BlobPurgeJobStopRequested(); + return; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + this.logger.BlobPurgeStarterRetry(ex); + try + { + await Task.Delay(RetryDelay, cancellationToken); + } + catch (OperationCanceledException) + { + return; + } + } + } + } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 2d55a979..0c13984a 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -13,9 +13,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; class BlobPurgeJob(ILogger logger) : TaskEntity { /// - /// Creates (or reactivates) the auto-purge job. Because the job is a per-task-hub singleton, this is - /// intentionally a no-op when the job is already so that extra - /// client processes racing to create it do not disturb the running job. + /// Creates (or reactivates) the auto-purge job. Because the job is a per-task-hub singleton, this does not + /// restart a job that is already so that extra client processes + /// racing to create it do not disturb the running job. It does still take the batch size in that case, which + /// is what lets a configuration change reach a job that is already running. /// /// The entity context. /// @@ -25,6 +26,14 @@ public void Create(TaskEntityContext context, int purgeBatchSize) { if (this.State.Status == BlobPurgeJobStatus.Active) { + // Deliberately not re-signalling Run: the orchestrator is already up, and starting a second one + // over a live one is destructive. The batch size is still taken, because this is the only path by + // which a changed configuration reaches an active job - the orchestrator re-reads it from here + // every cycle. Without this the value written by the very first Create would be the only one the + // job ever used, and a batch size the backend rejects would wedge it permanently. + this.State.PurgeBatchSize = purgeBatchSize; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + logger.BlobPurgeJobAlreadyRunning(context.Id.Key); return; } @@ -64,6 +73,47 @@ public void Run(TaskEntityContext context) this.State.LastModifiedAt = DateTimeOffset.UtcNow; } + /// + /// Stops the auto-purge job. + /// + /// + /// + /// The perpetual orchestrator is deliberately not terminated here, and this operation does not touch its + /// instance ID at all. The orchestrator reads this entity at the top of every cycle and exits on its own + /// once the job is no longer , so shutdown is cooperative: there is + /// no window in which one party terminates an orchestrator that the other believes is healthy, and the + /// in-flight cycle finishes rather than being cut off part-way through a batch of deletes. + /// + /// + /// , and + /// are preserved. They are the job's history and its + /// configuration, both of which are wanted if it is started again, and keeping CreatedAt is also what + /// distinguishes a stopped job from one that was never started. + /// + /// + /// The entity context. + public void Stop(TaskEntityContext context) + { + if (this.State.Status != BlobPurgeJobStatus.Active) + { + // Idempotent, and the repeat is the common case rather than the exception: a host with auto-purge + // disabled signals Stop on every start, including for an app that never enabled auto-purge at all. + // Returning here leaves the state exactly as it was found. + // + // This does not avoid materializing the entity - the framework persists entity state after every + // operation, so a stop signal to an entity that does not exist yet creates it with default state. + // What the guard preserves is LastModifiedAt: rewriting it on every host restart would destroy its + // only useful meaning, which is when the job actually stopped. + logger.BlobPurgeJobAlreadyStopped(context.Id.Key); + return; + } + + this.State.Status = BlobPurgeJobStatus.Pending; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.BlobPurgeJobStopped(context.Id.Key); + } + /// /// Records progress after a purge cycle completes. /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 8b40c970..a1f1eaf3 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -28,10 +28,13 @@ static partial class Logs [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge reported {count} purge result(s) to the backend.")] public static partial void BlobPurgeReportedResults(this ILogger logger, int count); + [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' stopped. The perpetual orchestrator is not terminated; it reads the job state at the start of its next cycle and exits on its own.")] + public static partial void BlobPurgeJobStopped(this ILogger logger, string? jobId); + [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] public static partial void BlobPurgeJobEnsured(this ILogger logger); - [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] + [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not reach the singleton job; retrying.")] public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; cause '{cause}', storage code '{storageCode}'. The backend reschedules the tombstone for a later attempt. The reported result carries the disposition alone, so this log is the only record of the cause.")] @@ -43,6 +46,12 @@ static partial class Logs [LoggerMessage(EventId = 821, Level = LogLevel.Warning, Message = "An externalized payload blob does not carry this store's ownership marker, so it was left untouched; the tombstone is still resolved. This is expected for payloads written before the marker shipped, and for blobs the store never created whose token text matches the payload token grammar.")] public static partial void BlobPurgeBlobNotStoreOwned(this ILogger logger); + [LoggerMessage(EventId = 822, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is already stopped; ignoring the stop request.")] + public static partial void BlobPurgeJobAlreadyStopped(this ILogger logger, string? jobId); + [LoggerMessage(EventId = 823, Level = LogLevel.Error, Message = "Blob payload auto-purge is enabled but the registered PayloadStore ('{storeType}') is not an Azure Blob payload store and cannot delete payloads. The auto-purge job was not started; externalized payloads will not be reclaimed. Register the Azure Blob payload store, or disable AutoPurge.")] public static partial void BlobPurgeStoreCannotDelete(this ILogger logger, string? storeType); + + [LoggerMessage(EventId = 824, Level = LogLevel.Information, Message = "Blob payload auto-purge is disabled, so a stop was requested for the singleton job. A job left running by an earlier configuration exits after its current cycle.")] + public static partial void BlobPurgeJobStopRequested(this ILogger logger); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs index 97c37e66..fe6fe99c 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs @@ -9,8 +9,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public enum BlobPurgeJobStatus { /// - /// The job has not been started yet. This is the default status of a freshly initialized entity, so it is - /// kept as the zero value to avoid a brand-new entity accidentally appearing active. + /// The job is not running. This is both the state of a job that has never been started and the resting + /// state of one that has been stopped, which distinguishes: it is + /// null only for a job that was never created. It is kept as the zero value so a brand-new entity does not + /// accidentally appear active. /// Pending, diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 4e964612..a4e455e4 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -57,9 +57,14 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator( - input.JobEntityId, nameof(BlobPurgeJob.Get), null); + input.JobEntityId, nameof(BlobPurgeJob.Get), input: null); if (state is null || state.Status != BlobPurgeJobStatus.Active) { @@ -67,9 +72,22 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator 0 ? state.PurgeBatchSize : batchSize; + List tombstones = await context.CallActivityAsync>( nameof(GetLargePayloadTombstonesActivity), - batchSize, + cycleBatchSize, new TaskOptions(PurgeActivityRetryPolicy)); if (tombstones is null || tombstones.Count == 0) diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs new file mode 100644 index 00000000..609cdcb9 --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class BlobPurgeJobOrchestratorTests +{ + static readonly EntityInstanceId JobEntityId = new(nameof(BlobPurgeJob), BlobPurgeConstants.JobId); + + readonly List requested = []; + readonly TestLogger logger = new(); + + [Fact] + public async Task RunAsync_UsesBatchSizeFromEntity_NotFromInput() + { + // Arrange - the orchestrator is perpetual, so its input is fixed at creation and carried verbatim + // through every continue-as-new. Reading the batch size from the input would pin the value written by + // the very first Create for the life of the job, which is what made a configuration change impossible + // to apply. The entity is the authority. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 777 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert + this.AssertNoCycleFailed(); + this.requested.Should().Equal(777); + } + + [Fact] + public async Task RunAsync_WhenEntityBatchSizeIsUnset_FallsBackToInput() + { + // Arrange - an entity written by an older build carries no batch size at all. Passing that zero to the + // fetch activity would ask the backend for nothing on every cycle: a silent, permanent stall. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 0 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert + this.AssertNoCycleFailed(); + this.requested.Should().Equal(100); + } + + [Fact] + public async Task RunAsync_WhenJobIsStopped_ExitsWithoutFetching() + { + // Arrange - Stop moves the entity off Active without touching this orchestrator, so the exit is the + // orchestrator's own decision on its own schedule. Nothing must be fetched on a stopped job. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending, PurgeBatchSize = 777 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert - an empty fetch list is also what a broken mock produces, so the stopping log is asserted + // too: it is the only evidence that the orchestrator read the state and chose to exit. + this.AssertNoCycleFailed(); + this.requested.Should().BeEmpty(); + this.logger.Logs.Should().Contain(entry => entry.Message.Contains("stopping")); + } + + /// + /// Guards against the whole test passing through the orchestrator's catch-all cycle handler, which would + /// leave every observation empty and make the assertions vacuous. + /// + void AssertNoCycleFailed() => + this.logger.Logs.Should().NotContain(entry => entry.Message.Contains("cycle for job")); + + /// + /// Builds a context whose first entity read returns and whose second returns a + /// stopped job, so the perpetual loop runs at most one cycle and then exits. Batch sizes passed to the + /// fetch activity are recorded. + /// + Mock ContextFor(BlobPurgeJobState first) + { + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + entities + .SetupSequence(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(first) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending }); + + context + .Setup(c => c.CallActivityAsync>( + It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, input, _) => this.requested.Add((int)input!)) + .ReturnsAsync([]); + + return context; + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index 2aea6a84..2e6a3fff 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -4,6 +4,8 @@ using FluentAssertions; using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Options; using Xunit; @@ -63,27 +65,36 @@ public async Task StartAsync_WhenStoreIsBlobStore_DoesNotShortCircuit() } [Fact] - public async Task StartAsync_WhenAutoPurgeDisabled_DoesNotResolveClientOrLog() + public async Task StartAsync_WhenAutoPurgeDisabled_SignalsJobToStop() { - // Arrange - auto-purge is off. Even with a delete-capable blob store, the starter must no-op silently: - // it is registered unconditionally, so the not-opted-in path is the common case and must not log or - // resolve a client. + // Arrange - auto-purge is off. Returning silently was the whole disable bug: the job is a perpetual + // orchestrator owned by the task hub, so one created while the flag was on keeps deleting blobs no + // matter how many hosts start with it off. The disable path has to reach the backend. BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); - Mock provider = new(); - TestLogger logger = new(); - BlobPurgeJobStarter starter = new( - provider.Object, - store, - OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = false }), - "test", - logger); // Act - await starter.StartAsync(CancellationToken.None); + (EntityInstanceId Id, string Operation)? signal = await SignalFromDisabledStarterAsync(store); - // Assert - returned before resolving a client and without logging anything at all. - provider.Verify(p => p.GetClient(It.IsAny()), Times.Never); - logger.Logs.Should().BeEmpty(); + // Assert + signal.Should().NotBeNull("the disable path must tell the job to stop"); + signal!.Value.Id.Should().Be(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); + signal.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndStoreCannotDelete_StillSignalsJobToStop() + { + // Arrange - the store-capability gate deliberately does not apply to the disable path. Stopping a job + // requires no ability to delete anything, and a user who has switched to a store that cannot delete is + // precisely the user whose still-running job must be stopped. + + // Act + (EntityInstanceId Id, string Operation)? signal = + await SignalFromDisabledStarterAsync(new NonDeletingPayloadStore()); + + // Assert + signal.Should().NotBeNull("a store that cannot delete must not block stopping the job"); + signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); } [Fact] @@ -139,6 +150,44 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag return monitor.Object; } + /// + /// Runs a starter with auto-purge disabled and returns the entity signal it emitted, or null if none was + /// emitted within the timeout. The signal runs on a background task, so it is awaited rather than assumed. + /// + static async Task<(EntityInstanceId Id, string Operation)?> SignalFromDisabledStarterAsync(PayloadStore store) + { + Mock entities = new("test"); + TaskCompletionSource<(EntityInstanceId Id, string Operation)> signalled = new(); + entities + .Setup(e => e.SignalEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback( + (id, operation, _, _, _) => signalled.TrySetResult((id, operation))) + .Returns(Task.CompletedTask); + + Mock client = new("test"); + client.Setup(c => c.Entities).Returns(entities.Object); + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(client.Object); + + BlobPurgeJobStarter starter = new( + provider.Object, + store, + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = false }), + "test", + new TestLogger()); + + await starter.StartAsync(CancellationToken.None); + Task completed = await Task.WhenAny(signalled.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await starter.StopAsync(CancellationToken.None); + + return completed == signalled.Task ? await signalled.Task : null; + } + sealed class NonDeletingPayloadStore : PayloadStore { // DeleteAsync is intentionally NOT overridden: the base PayloadStore.DeleteAsync throws diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 2117299c..8a6ca306 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -3,6 +3,7 @@ using FluentAssertions; using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Entities; using Microsoft.DurableTask.Entities.Tests; using Xunit; @@ -31,12 +32,24 @@ public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() state.PurgeBatchSize.Should().Be(250); state.CreatedAt.Should().NotBeNull(); state.LastModifiedAt.Should().NotBeNull(); + + // Starting the job means signalling Run. Asserted here so that the Times.Never assertion in the + // already-active test below is meaningful rather than passing because the mock records nothing. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Once); } [Fact] - public async Task Create_WhenAlreadyActive_IsNoOp() + public async Task Create_WhenAlreadyActive_UpdatesBatchSizeWithoutRestarting() { - // Arrange + // Arrange - the job is already running and the configured batch size has changed. Create is the only + // path by which a new batch size can reach an active job, so it must be taken even though the job is + // not restarted. BlobPurgeJobState existing = new() { Status = BlobPurgeJobStatus.Active, @@ -50,11 +63,83 @@ public async Task Create_WhenAlreadyActive_IsNoOp() // Act await this.job.RunAsync(operation); - // Assert - status stays Active and the original batch size is retained, proving the create no-op'd. + // Assert BlobPurgeJobState state = Assert.IsType( operation.State.GetState(typeof(BlobPurgeJobState))); state.Status.Should().Be(BlobPurgeJobStatus.Active); - state.PurgeBatchSize.Should().Be(100); + state.PurgeBatchSize.Should().Be(999); + + // Run is not re-signalled: the orchestrator is already up, and scheduling a second one over a live one + // would terminate and replace it mid-work. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Stop_WhenActive_MovesToPendingAndKeepsHistory() + { + // Arrange - a running job. Stopping must not discard the configuration or the progress counters: they + // are wanted if the job is started again, and CreatedAt is what distinguishes a stopped job from one + // that was never started. + DateTimeOffset createdAt = DateTimeOffset.UtcNow.AddDays(-3); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + CreatedAt = createdAt, + PurgedCount = 17, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Stop), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Pending); + state.CreatedAt.Should().Be(createdAt); + state.PurgedCount.Should().Be(17); + state.PurgeBatchSize.Should().Be(250); + state.LastModifiedAt.Should().NotBeNull(); + } + + [Fact] + public async Task Stop_WhenNotActive_LeavesStateUntouched() + { + // Arrange - a job that is already stopped. Every host with auto-purge disabled signals Stop on each + // start, so the repeat is the common case; rewriting LastModifiedAt each time would destroy its only + // useful meaning, which is when the job actually stopped. + DateTimeOffset stoppedAt = DateTimeOffset.UtcNow.AddHours(-6); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Pending, + CreatedAt = DateTimeOffset.UtcNow.AddDays(-3), + LastModifiedAt = stoppedAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Stop), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Pending); + state.LastModifiedAt.Should().Be(stoppedAt); + state.PurgeBatchSize.Should().Be(250); } [Fact] From 7d409ab9d71ae557a3350e330e2e0d246f55aeb9 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 20:28:07 -0700 Subject: [PATCH 28/47] Read the purge job state before signalling Stop The disable path signalled Stop unconditionally. Entity state is persisted after every operation, so that signal created the job entity - in its default, never-started state - for every app that externalizes payloads without ever enabling auto-purge, and persisted a fresh write on every host start for an app whose job was already stopped. The starter now reads the job through the client first and signals only when a job exists and is Active. The read is a query against the instance store rather than an entity operation, so it never dispatches to the entity and cannot materialize one; that is the property that makes it different from the Get operation the orchestrator calls. Three things the pre-check must not break, and does not: - The Stop guard in the entity stays. The pre-check is read-then-signal across two round trips with nothing holding the state still, so it is racy by construction; the guard is what makes losing that race harmless. Its comment now states the race as the justification rather than the steady state, which the pre-check has just made false. - A read that fails falls through to signalling in the same iteration rather than being retried. Every 'entities are not supported' gate in this SDK lives on the DurableTaskClient.Entities property, not on individual methods, so a client that cannot answer the query cannot receive the signal either - retrying a permanent failure would spin forever and the running job would never be told to stop. - EventId 824 now fires only when a stop is actually requested. New EventIds 825 (no running job found) and 826 (state unreadable, stopping anyway) cover the paths where it would otherwise have asserted something that did not happen. The status is reached through EntityMetadata.IncludesState rather than by reading .State, which throws when the metadata carries none - reachable for an entity whose state was cleared but which still reports as existing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 76 ++++++++-- .../AutoPurge/Entity/BlobPurgeJob.cs | 19 ++- .../AzureBlobPayloads/AutoPurge/Logs.cs | 6 + .../AutoPurge/BlobPurgeJobStarterTests.cs | 141 +++++++++++++++--- 4 files changed, 207 insertions(+), 35 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 477eac9b..db8cd248 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -3,6 +3,7 @@ using DurableTask.Core.Exceptions; using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -13,10 +14,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Client-side hosted service that reconciles the per-task-hub singleton blob payload auto-purge job with the /// resolved options. It is registered unconditionally by UseExternalizedPayloads and decides what to do at -/// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, asks the job -/// to stop when it is disabled, and no-ops with an error log when the registered store cannot delete. It never -/// blocks host startup - the work runs on a background task that retries until the backend is reachable. The -/// job is a per-task-hub singleton, so racing client processes converge on the same result. +/// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, stops a +/// running job when it is disabled, and no-ops with an error log when the registered store cannot delete. It +/// never blocks host startup - the work runs on a background task that retries until the backend is reachable. +/// The job is a per-task-hub singleton, so racing client processes converge on the same result. /// sealed class BlobPurgeJobStarter : IHostedService, IDisposable { @@ -68,16 +69,15 @@ public Task StartAsync(CancellationToken cancellationToken) { // Turning auto-purge off has to reach the backend to mean anything. The job is a perpetual // orchestrator owned by the task hub, not by this process, so a job created while the flag was on - // keeps deleting blobs forever no matter how many hosts start with it off. Ask it to stop. + // keeps deleting blobs forever no matter how many hosts start with it off. Stop a running one. // // The store gate below deliberately does NOT apply to this path. Stopping a job requires no ability // to delete anything, and a user who has switched to a store that cannot delete is precisely // someone who needs the job stopped; that gate's error is worded for the enabled case and would be // a false alarm here. // - // One deliberate cost: entity state is persisted after every operation, so this signal creates the - // job entity in its default, not-running state for apps that never enabled auto-purge. That is a - // single small entity per task hub, and it is the same one the job would use if it were enabled. + // The job's state is read before anything is signalled, so an app that never enabled auto-purge + // signals nothing and no entity is created for it. this.cts = new CancellationTokenSource(); this.backgroundTask = Task.Run(() => this.SignalJobStopAsync(this.cts.Token), CancellationToken.None); return Task.CompletedTask; @@ -237,10 +237,68 @@ async Task SignalJobStopAsync(CancellationToken cancellationToken) // which must never happen for a feature the user has turned off. DurableTaskClient client = this.clientProvider.GetClient(this.builderName); + // Held in a local so the query and the signal below are issued against the same object. Both go + // through this one property, which is also where every "entities are not supported" gate in this + // SDK lives, so a client that cannot answer the query cannot receive the signal either. + DurableEntityClient entities = client.Entities; + + // Defaults to true so that every path which fails to establish the job's state falls through to + // signalling. Signalling is the correctness-critical outcome - a job left running keeps deleting + // payloads - while skipping it is only an optimization, so the optimization must never be the + // reason the stop does not happen. + bool stopNeeded = true; + + try + { + // A query against the instance store, not an entity operation: it never dispatches to the + // entity, so it cannot materialize one. That is the whole point of reading through the + // client here instead of relying on the entity's own idempotence. Signalling Stop + // unconditionally would create the job entity - with default, never-started state - for + // every app that externalizes payloads without auto-purge, and would persist a fresh write + // on every host start for an app whose job is already stopped. + // The three-argument overload is called deliberately. The shorter GetEntityAsync(id, + // cancellation) is a virtual forwarder that supplies includeState itself, and binding to a + // forwarder rather than to the abstract method is what made an earlier set of tests in this + // feature pass against a mock that was never wired up. Naming includeState also states the + // requirement directly: this call exists to read the status, so metadata alone is useless. + EntityMetadata? metadata = await entities + .GetEntityAsync(this.entityId, includeState: true, cancellationToken); + + // IncludesState is checked rather than reading metadata.State directly: State throws when + // the metadata carries none, and an entity whose state has been cleared still reports as + // existing until entity storage is cleaned. Such an entity has no running job, so it is + // treated the same as an absent one. + BlobPurgeJobState? state = metadata is { IncludesState: true } ? metadata.State : null; + + stopNeeded = state?.Status == BlobPurgeJobStatus.Active; + + if (!stopNeeded) + { + this.logger.BlobPurgeJobNotRunning(); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + // Deliberately not rethrown into the retry loop. Retrying would be right for a transient + // failure but fatal for a permanent one: a client that never answers this query would keep + // the loop spinning and the running job would never be told to stop. Falling through costs + // at most the entity write this pre-check exists to avoid. + this.logger.BlobPurgeJobStateUnknown(ex); + } + + if (!stopNeeded) + { + return; + } + // Signalled rather than driven through the bridge orchestration: unlike Create, Stop needs no // fixed-instance dedupe. It is idempotent in the entity, so concurrent hosts converge on the // same state and a repeat costs nothing beyond the signal itself. - await client.Entities.SignalEntityAsync( + await entities.SignalEntityAsync( this.entityId, nameof(BlobPurgeJob.Stop), null, diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 0c13984a..fb9303e2 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -96,14 +96,19 @@ public void Stop(TaskEntityContext context) { if (this.State.Status != BlobPurgeJobStatus.Active) { - // Idempotent, and the repeat is the common case rather than the exception: a host with auto-purge - // disabled signals Stop on every start, including for an app that never enabled auto-purge at all. - // Returning here leaves the state exactly as it was found. + // Load-bearing, and NOT redundant with the client-side pre-check in BlobPurgeJobStarter that + // normally prevents this operation from being signalled at all. That pre-check reads the job state + // and then signals: two separate round trips with nothing holding the state still between them, so + // the job can stop - or never have existed - in the gap. This guard is what makes losing that race + // harmless, which is the only reason the pre-check is allowed to be a plain read. Deleting it + // because "the caller already checked" would reintroduce exactly the window it was written to + // absorb. Concurrent hosts signalling at once land here for the same reason. // - // This does not avoid materializing the entity - the framework persists entity state after every - // operation, so a stop signal to an entity that does not exist yet creates it with default state. - // What the guard preserves is LastModifiedAt: rewriting it on every host restart would destroy its - // only useful meaning, which is when the job actually stopped. + // Returning here also leaves the state exactly as it was found. That preserves LastModifiedAt, + // whose only useful meaning is when the job actually stopped; rewriting it on a redundant stop + // would destroy that. It does not avoid materializing the entity - the framework persists entity + // state after every operation, so a stop signal to an entity that does not exist yet creates it + // with default state. Not creating it is the pre-check's job, not this guard's. logger.BlobPurgeJobAlreadyStopped(context.Id.Key); return; } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index a1f1eaf3..ce1ba658 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -54,4 +54,10 @@ static partial class Logs [LoggerMessage(EventId = 824, Level = LogLevel.Information, Message = "Blob payload auto-purge is disabled, so a stop was requested for the singleton job. A job left running by an earlier configuration exits after its current cycle.")] public static partial void BlobPurgeJobStopRequested(this ILogger logger); + + [LoggerMessage(EventId = 825, Level = LogLevel.Debug, Message = "Blob payload auto-purge is disabled and no running singleton job was found, so no stop was requested.")] + public static partial void BlobPurgeJobNotRunning(this ILogger logger); + + [LoggerMessage(EventId = 826, Level = LogLevel.Warning, Message = "Blob payload auto-purge is disabled but the singleton job's state could not be read; requesting a stop anyway. Stopping a job that may be deleting payloads matters more than avoiding a redundant stop request, which the job ignores.")] + public static partial void BlobPurgeJobStateUnknown(this ILogger logger, Exception exception); } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index 2e6a3fff..a6a3db91 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -65,20 +65,94 @@ public async Task StartAsync_WhenStoreIsBlobStore_DoesNotShortCircuit() } [Fact] - public async Task StartAsync_WhenAutoPurgeDisabled_SignalsJobToStop() + public async Task StartAsync_WhenAutoPurgeDisabledAndJobIsActive_SignalsJobToStop() { - // Arrange - auto-purge is off. Returning silently was the whole disable bug: the job is a perpetual - // orchestrator owned by the task hub, so one created while the flag was on keeps deleting blobs no - // matter how many hosts start with it off. The disable path has to reach the backend. + // Arrange - auto-purge is off while a job is running. Returning silently was the whole disable bug: the + // job is a perpetual orchestrator owned by the task hub, so one created while the flag was on keeps + // deleting blobs no matter how many hosts start with it off. The disable path has to reach the backend. BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); // Act - (EntityInstanceId Id, string Operation)? signal = await SignalFromDisabledStarterAsync(store); + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = + await RunDisabledStarterAsync(store, () => MetadataFor(BlobPurgeJobStatus.Active)); // Assert - signal.Should().NotBeNull("the disable path must tell the job to stop"); - signal!.Value.Id.Should().Be(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); - signal.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + run.Signal.Should().NotBeNull("the disable path must tell a running job to stop"); + run.Signal!.Value.Id.Should().Be(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); + run.Signal.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndNoJobExists_DoesNotSignal() + { + // Arrange - the common case by far: an app that externalizes payloads and never enabled auto-purge. + // Signalling regardless would create the job entity, because the framework persists entity state after + // every operation, so an app that never used the feature would still carry one entity per task hub. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + + // Act + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = + await RunDisabledStarterAsync(store, () => null); + + // Assert - Reads is the positive control. A null signal is also what a mock that was never reached + // produces, so the assertion that nothing was signalled is only meaningful alongside proof that the + // code did run and did ask. The same wiring signals when the job is active, which the test above pins. + run.Reads.Should().Be(1, "the starter must consult the job state rather than skipping the path entirely"); + run.Signal.Should().BeNull("there is no job to stop, so nothing may be created"); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndJobIsAlreadyStopped_DoesNotSignal() + { + // Arrange - a job that has already been stopped needs nothing. The entity would ignore the signal, but + // the framework still persists state on every operation, so signalling would write on every host start. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + + // Act + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = + await RunDisabledStarterAsync(store, () => MetadataFor(BlobPurgeJobStatus.Pending)); + + // Assert + run.Reads.Should().Be(1); + run.Signal.Should().BeNull("a job that is not active must not be signalled on every restart"); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndEntityHasNoState_DoesNotSignal() + { + // Arrange - an entity whose state has been cleared still reports as existing until entity storage is + // cleaned, and its metadata carries no state at all. Reading EntityMetadata.State in that condition + // throws, so the status has to be reached through IncludesState. Such an entity has no running job. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + + // Act + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = await RunDisabledStarterAsync( + store, + () => new EntityMetadata( + new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId))); + + // Assert + run.Reads.Should().Be(1); + run.Signal.Should().BeNull("an entity with no state has no running job"); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndStateCannotBeRead_SignalsJobToStopAnyway() + { + // Arrange - the pre-check is an optimization; stopping the job is the correctness-critical outcome. A + // client that cannot answer the query must not be able to prevent the stop, or the optimization's own + // failure mode would defeat the fix it is attached to. Retrying instead of falling through would be + // just as bad for a permanent failure: the loop would spin and the running job would never be told. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + + // Act + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = await RunDisabledStarterAsync( + store, + () => throw new NotSupportedException("entity queries are not supported")); + + // Assert + run.Signal.Should().NotBeNull("an unreadable state must not suppress the stop"); + run.Signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); } [Fact] @@ -89,12 +163,12 @@ public async Task StartAsync_WhenAutoPurgeDisabledAndStoreCannotDelete_StillSign // precisely the user whose still-running job must be stopped. // Act - (EntityInstanceId Id, string Operation)? signal = - await SignalFromDisabledStarterAsync(new NonDeletingPayloadStore()); + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = await RunDisabledStarterAsync( + new NonDeletingPayloadStore(), () => MetadataFor(BlobPurgeJobStatus.Active)); // Assert - signal.Should().NotBeNull("a store that cannot delete must not block stopping the job"); - signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + run.Signal.Should().NotBeNull("a store that cannot delete must not block stopping the job"); + run.Signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); } [Fact] @@ -151,13 +225,39 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag } /// - /// Runs a starter with auto-purge disabled and returns the entity signal it emitted, or null if none was - /// emitted within the timeout. The signal runs on a background task, so it is awaited rather than assumed. + /// Builds entity metadata carrying a job in the given status. + /// + static EntityMetadata MetadataFor(BlobPurgeJobStatus status) => + new( + new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId), + new BlobPurgeJobState { Status = status }); + + /// + /// Runs a starter with auto-purge disabled, using as the job state the client + /// reports (it may return null for an absent entity, or throw to simulate a client that cannot answer). + /// Returns the entity signal the starter emitted, if any, and how many times the state was read. /// - static async Task<(EntityInstanceId Id, string Operation)?> SignalFromDisabledStarterAsync(PayloadStore store) + static async Task<(( EntityInstanceId Id, string Operation)? Signal, int Reads)> RunDisabledStarterAsync( + PayloadStore store, Func?> read) { Mock entities = new("test"); - TaskCompletionSource<(EntityInstanceId Id, string Operation)> signalled = new(); + TaskCompletionSource readCalled = new(); + (EntityInstanceId Id, string Operation)? signal = null; + int reads = 0; + + // The three-argument overload is set up because that is the one the starter calls. The shorter + // GetEntityAsync(id, cancellation) is virtual, so a mock overrides it instead of forwarding, and a + // setup placed on the abstract method alone would silently never match. + entities + .Setup(e => e.GetEntityAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => + { + reads++; + readCalled.TrySetResult(true); + return Task.FromResult(read()); + }); + entities .Setup(e => e.SignalEntityAsync( It.IsAny(), @@ -166,7 +266,7 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag It.IsAny(), It.IsAny())) .Callback( - (id, operation, _, _, _) => signalled.TrySetResult((id, operation))) + (id, operation, _, _, _) => signal = (id, operation)) .Returns(Task.CompletedTask); Mock client = new("test"); @@ -181,11 +281,14 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag "test", new TestLogger()); + // The work runs on a background task, so wait for the read before shutting down. StopAsync then waits + // for that task to finish, which makes the observation deterministic: by the time it returns the + // starter has either signalled or decided not to, rather than being timed out mid-decision. await starter.StartAsync(CancellationToken.None); - Task completed = await Task.WhenAny(signalled.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await Task.WhenAny(readCalled.Task, Task.Delay(TimeSpan.FromSeconds(30))); await starter.StopAsync(CancellationToken.None); - return completed == signalled.Task ? await signalled.Task : null; + return (signal, reads); } sealed class NonDeletingPayloadStore : PayloadStore From 7e0d0ed5bc9385352f77941751c8d51b87f154f8 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 21:09:26 -0700 Subject: [PATCH 29/47] Self-heal the blob purge job by re-signalling Run on every host start Create's already-active branch previously did not re-signal Run, on the stated grounds that starting a second orchestrator over a live one is destructive. That premise is false. An entity-initiated start carries no reuse policy - neither TaskEntityShim.ScheduleNewOrchestration nor the StartNewOrchestrationAction proto message has a field for one - so the backend resolves it atomically: OkToPurge refuses to replace an instance that is not IsCompleted, and the start message is discarded; a completed, terminated, failed or canceled instance is purged and replaced. Healthy jobs are therefore undisturbed and dead ones are recovered, with no check-then-act on our side. Create now re-signals Run after updating the batch size, which makes the job recover from a dead orchestrator on the next host start. Also corrects two comments and one log message that asserted the old, false premise, including a longer restatement of it in BlobPurgeJobStarter that described manual recovery as the only option. Run had no test coverage; both its guarded and unguarded paths are now pinned, since the guard is what makes a Stop win against an in-flight Create. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 17 +-- .../AutoPurge/Entity/BlobPurgeJob.cs | 42 +++++-- .../AzureBlobPayloads/AutoPurge/Logs.cs | 2 +- .../AutoPurge/BlobPurgeJobTests.cs | 107 ++++++++++++++++-- 4 files changed, 140 insertions(+), 28 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index db8cd248..d43e47c5 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -172,14 +172,15 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // permanently; making it replaceable means the next host start re-runs Create, which finds the // entity not Active and rebuilds the job. // - // This deliberately does NOT recover the case where the perpetual orchestrator dies while the - // entity is still Active: the bridge re-runs, Create no-ops on the Active state, and the - // orchestrator stays down. That is a deliberate non-goal rather than an oversight. The only - // available restart is signalling the entity's Run, which schedules the orchestrator on the - // entity path with no dedupe policy available to it, so signalling a healthy orchestrator would - // terminate and replace it mid-work. Deciding it is safe to signal requires a check-then-act on - // the orchestrator's status, which is exactly the non-atomic pattern this call replaced. - // Recovery is therefore a supervised manual step. + // This also recovers the case where the perpetual orchestrator dies while the entity is still + // Active. Create leaves the entity's status alone in that case, but it re-signals Run, and the + // resulting start is resolved by the backend: discarded outright while the orchestrator is + // alive, and allowed to replace it once it has completed, terminated, failed or been canceled. + // So a healthy job is left untouched and a dead one is rebuilt, without this side ever having to + // ask which of the two it is looking at. + // + // Recovery is bounded by host starts rather than being continuous: nothing re-signals Run + // between them, so an orchestrator that dies mid-lifetime stays down until the next Create. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index fb9303e2..8c4dd893 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -13,10 +13,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; class BlobPurgeJob(ILogger logger) : TaskEntity { /// - /// Creates (or reactivates) the auto-purge job. Because the job is a per-task-hub singleton, this does not - /// restart a job that is already so that extra client processes - /// racing to create it do not disturb the running job. It does still take the batch size in that case, which - /// is what lets a configuration change reach a job that is already running. + /// Creates the auto-purge job, and starts its orchestrator if one is not already running. Because the job is + /// a per-task-hub singleton, client processes racing to create it converge on the same result rather than + /// disturbing a running job. It also takes the batch size when the job is already + /// , which is what lets a configuration change reach a running job. /// /// The entity context. /// @@ -26,15 +26,28 @@ public void Create(TaskEntityContext context, int purgeBatchSize) { if (this.State.Status == BlobPurgeJobStatus.Active) { - // Deliberately not re-signalling Run: the orchestrator is already up, and starting a second one - // over a live one is destructive. The batch size is still taken, because this is the only path by - // which a changed configuration reaches an active job - the orchestrator re-reads it from here - // every cycle. Without this the value written by the very first Create would be the only one the - // job ever used, and a batch size the backend rejects would wedge it permanently. + // The batch size is taken because this is the only path by which a changed configuration reaches an + // active job - the orchestrator re-reads it from here every cycle. Without this the value written by + // the very first Create would be the only one the job ever used, and a batch size the backend + // rejects would wedge it permanently. this.State.PurgeBatchSize = purgeBatchSize; this.State.LastModifiedAt = DateTimeOffset.UtcNow; logger.BlobPurgeJobAlreadyRunning(context.Id.Key); + + // Run is re-signalled even though the job is already active, and this is what makes the job + // self-heal: Create runs on every host start, so a job whose orchestrator has died is rebuilt at the + // next one. The signal is deliberately blind. An entity-initiated start carries no reuse policy, so + // the backend decides its fate: it discards the start while the target instance exists in any + // non-completed status, and purges and replaces it once the instance has completed, terminated, + // failed or been canceled (its OkToPurge / IsCompleted rule). A healthy orchestrator is therefore + // left strictly alone and only a dead one is replaced. + // + // Being blind is the point, not a shortcut. The backend reaches that decision atomically within one + // partition operation, so delegating it removes the race entirely. Checking the orchestrator's + // status here and signalling only when it looked dead would reintroduce the window in which it dies + // - or recovers - between the read and the signal, which is strictly worse than not asking. + context.SignalEntity(context.Id, nameof(this.Run)); return; } @@ -51,9 +64,16 @@ public void Create(TaskEntityContext context, int purgeBatchSize) } /// - /// Starts the purge orchestrator if the job is active. Uses a fixed orchestrator instance ID so only one - /// orchestrator ever runs for the singleton job. + /// Starts the purge orchestrator if the job is active. /// + /// + /// The orchestrator runs under a fixed instance ID, which is what keeps the singleton a singleton. No reuse + /// policy is passed, and none can be: the entity's start action has no field to carry one, so anything set + /// here would be dropped before it reached the wire. That default is the behaviour the job relies on rather + /// than an omission - the backend discards a start aimed at an instance that already exists in a + /// non-completed status, and replaces the instance only once it has completed, terminated, failed or been + /// canceled. Signalling this operation is therefore always safe, whatever the orchestrator is doing. + /// /// The entity context. public void Run(TaskEntityContext context) { diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index ce1ba658..b4042edd 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -13,7 +13,7 @@ static partial class Logs [LoggerMessage(EventId = 810, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' created.")] public static partial void BlobPurgeJobCreated(this ILogger logger, string? jobId); - [LoggerMessage(EventId = 811, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' is already running; ignoring the create request.")] + [LoggerMessage(EventId = 811, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' is already active. Its batch size was updated from the current configuration, and its orchestrator was re-signalled, which starts one only if none is running.")] public static partial void BlobPurgeJobAlreadyRunning(this ILogger logger, string? jobId); [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 8a6ca306..8b61237f 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -33,8 +33,9 @@ public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() state.CreatedAt.Should().NotBeNull(); state.LastModifiedAt.Should().NotBeNull(); - // Starting the job means signalling Run. Asserted here so that the Times.Never assertion in the - // already-active test below is meaningful rather than passing because the mock records nothing. + // Starting the job means signalling Run, and this path has always done so. Asserted explicitly because + // the already-active path now signals Run too, which makes this the case that would silently stop being + // covered if the two branches were ever collapsed. Mock.Get(operation.Context).Verify( c => c.SignalEntity( It.IsAny(), @@ -45,11 +46,10 @@ public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() } [Fact] - public async Task Create_WhenAlreadyActive_UpdatesBatchSizeWithoutRestarting() + public async Task Create_WhenAlreadyActive_UpdatesBatchSizeAndReSignalsRun() { // Arrange - the job is already running and the configured batch size has changed. Create is the only - // path by which a new batch size can reach an active job, so it must be taken even though the job is - // not restarted. + // path by which a new batch size can reach an active job, so it must be taken. BlobPurgeJobState existing = new() { Status = BlobPurgeJobStatus.Active, @@ -69,17 +69,108 @@ public async Task Create_WhenAlreadyActive_UpdatesBatchSizeWithoutRestarting() state.Status.Should().Be(BlobPurgeJobStatus.Active); state.PurgeBatchSize.Should().Be(999); - // Run is not re-signalled: the orchestrator is already up, and scheduling a second one over a live one - // would terminate and replace it mid-work. + // Run is re-signalled even though the job is already active. That is what lets a job whose orchestrator + // has died be rebuilt at the next host start, and it is safe because the backend discards the resulting + // start while the orchestrator is alive rather than replacing it. Mock.Get(operation.Context).Verify( c => c.SignalEntity( It.IsAny(), - It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Create_WhenAlreadyActive_SignalsNothingOtherThanRun() + { + // Arrange - pins that re-signalling Run is the only signal the already-active path emits. Verifying the + // Run signal alone would still pass if a second, different signal were added beside it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 100, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 999); + + // Act + await this.job.RunAsync(operation); + + // Assert - the Times.Once above is the positive control for this Times.Never: both target the same + // four-argument overload on the same mock, so this cannot be passing because nothing was recorded. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + It.Is(name => name != nameof(BlobPurgeJob.Run)), It.IsAny(), It.IsAny()), Times.Never); } + [Fact] + public async Task Run_WhenActive_SchedulesOrchestratorAtTheFixedInstanceId() + { + // Arrange - the fixed instance ID is the mechanism the whole restart story rests on. It is what lets the + // backend recognize a start as targeting the existing orchestrator, and therefore discard it while that + // orchestrator is alive instead of running a second one alongside it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + Mock.Get(operation.Context) + .Setup(c => c.Id) + .Returns(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); + + // Act + await this.job.RunAsync(operation); + + // Assert + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.Is(o => + o.InstanceId == BlobPurgeConstants.GetOrchestratorInstanceId(BlobPurgeConstants.JobId))), + Times.Once); + } + + [Fact] + public async Task Run_WhenNotActive_SchedulesNothing() + { + // Arrange - a Run signal arriving after the job was stopped. Create signals Run rather than starting the + // orchestrator itself, so a Stop landing between the two leaves this signal in flight against a job that + // must no longer purge. This guard is what makes the stop win instead of the stale signal restarting it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Pending, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert - the Times.Once above is the positive control: same mocked type, same three-argument overload, + // so this cannot be passing merely because the mock records nothing. + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + [Fact] public async Task Stop_WhenActive_MovesToPendingAndKeepsHistory() { From 720e028b03b57df48f7a410b4bcfa8091f7761fa Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 21:32:41 -0700 Subject: [PATCH 30/47] Make LastModifiedAt mean a real change, and drop internal backend symbol names LastModifiedAt was written by five operations, read by none, and two of those writes fired on every host start - Create's already-active branch and Run, the latter on every start since Run began being re-signalled. The field had degenerated into "the most recent host start, or the most recent non-empty purge, whichever is later", which also made Stop's idempotence guard vacuous: it deliberately preserves the field, but the next host start overwrote it anyway. Run no longer writes it - it schedules an orchestrator and changes nothing about the job - and Create's already-active branch writes it only when the batch size actually differs. An entity from a build predating the field carries zero, which differs from any configured size, so the first Create after an upgrade still repairs it. The field now means what its documentation says: the last time the job was started, stopped, reconfigured, or recorded a non-zero purge. Entity state is persisted after every operation regardless, so nothing about persistence changes. Also removes the names of two private backend methods from a comment. This repository is public and cannot reference them, so they were both a leak and certain to rot; the observable behaviour they described is kept. A sweep of the rest of the diff found no other instance outside the proto, whose comments are copied verbatim from the canonical contract. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Entity/BlobPurgeJob.cs | 46 ++++++--- .../AutoPurge/Models/BlobPurgeJobState.cs | 9 +- .../AutoPurge/BlobPurgeJobTests.cs | 98 ++++++++++++++++++- 3 files changed, 134 insertions(+), 19 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 8c4dd893..ce2199f7 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -30,8 +30,17 @@ public void Create(TaskEntityContext context, int purgeBatchSize) // active job - the orchestrator re-reads it from here every cycle. Without this the value written by // the very first Create would be the only one the job ever used, and a batch size the backend // rejects would wedge it permanently. - this.State.PurgeBatchSize = purgeBatchSize; - this.State.LastModifiedAt = DateTimeOffset.UtcNow; + // + // Written only when the value actually differs, which is what keeps LastModifiedAt tracking real + // configuration changes. Create runs on every host start, so an unconditional write would reduce + // the field to "time of the last host start". An entity written by a build that predates this + // field carries zero, which differs from any configured size, so the first Create after an upgrade + // still repairs it. + if (this.State.PurgeBatchSize != purgeBatchSize) + { + this.State.PurgeBatchSize = purgeBatchSize; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } logger.BlobPurgeJobAlreadyRunning(context.Id.Key); @@ -40,13 +49,13 @@ public void Create(TaskEntityContext context, int purgeBatchSize) // next one. The signal is deliberately blind. An entity-initiated start carries no reuse policy, so // the backend decides its fate: it discards the start while the target instance exists in any // non-completed status, and purges and replaces it once the instance has completed, terminated, - // failed or been canceled (its OkToPurge / IsCompleted rule). A healthy orchestrator is therefore - // left strictly alone and only a dead one is replaced. + // failed or been canceled. A healthy orchestrator is therefore left strictly alone and only a dead + // one is replaced. // - // Being blind is the point, not a shortcut. The backend reaches that decision atomically within one - // partition operation, so delegating it removes the race entirely. Checking the orchestrator's - // status here and signalling only when it looked dead would reintroduce the window in which it dies - // - or recovers - between the read and the signal, which is strictly worse than not asking. + // Being blind is the point, not a shortcut. The backend reaches that decision atomically, so + // delegating it removes the race entirely. Checking the orchestrator's status here and signalling + // only when it looked dead would reintroduce the window in which it dies - or recovers - between + // the read and the signal, which is strictly worse than not asking. context.SignalEntity(context.Id, nameof(this.Run)); return; } @@ -67,12 +76,19 @@ public void Create(TaskEntityContext context, int purgeBatchSize) /// Starts the purge orchestrator if the job is active. /// /// + /// /// The orchestrator runs under a fixed instance ID, which is what keeps the singleton a singleton. No reuse /// policy is passed, and none can be: the entity's start action has no field to carry one, so anything set /// here would be dropped before it reached the wire. That default is the behaviour the job relies on rather /// than an omission - the backend discards a start aimed at an instance that already exists in a /// non-completed status, and replaces the instance only once it has completed, terminated, failed or been /// canceled. Signalling this operation is therefore always safe, whatever the orchestrator is doing. + /// + /// + /// This operation deliberately writes no state. It schedules an orchestrator and nothing more, and it runs + /// on every host start, so touching here would overwrite a + /// real change with the time of a start that changed nothing. + /// /// /// The entity context. public void Run(TaskEntityContext context) @@ -89,8 +105,6 @@ public void Run(TaskEntityContext context) new TaskName(nameof(BlobPurgeJobOrchestrator)), new BlobPurgeJobRunRequest(context.Id, this.State.PurgeBatchSize), startOrchestrationOptions); - - this.State.LastModifiedAt = DateTimeOffset.UtcNow; } /// @@ -124,11 +138,13 @@ public void Stop(TaskEntityContext context) // because "the caller already checked" would reintroduce exactly the window it was written to // absorb. Concurrent hosts signalling at once land here for the same reason. // - // Returning here also leaves the state exactly as it was found. That preserves LastModifiedAt, - // whose only useful meaning is when the job actually stopped; rewriting it on a redundant stop - // would destroy that. It does not avoid materializing the entity - the framework persists entity - // state after every operation, so a stop signal to an entity that does not exist yet creates it - // with default state. Not creating it is the pre-check's job, not this guard's. + // Returning here also leaves the state exactly as it was found, which is what keeps + // LastModifiedAt meaning "when this job stopped" rather than "when a stop was last signalled at + // it". That only holds because no other operation rewrites the field on a no-op either: Run never + // writes it, and Create rewrites it only when the batch size actually differs. Breaking either of + // those breaks this too. It does not avoid materializing the entity - the framework persists + // entity state after every operation, so a stop signal to an entity that does not exist yet + // creates it with default state. Not creating it is the pre-check's job, not this guard's. logger.BlobPurgeJobAlreadyStopped(context.Id.Key); return; } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs index 8bd4cdd5..0eee8dd2 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs @@ -19,8 +19,15 @@ public sealed class BlobPurgeJobState public DateTimeOffset? CreatedAt { get; set; } /// - /// Gets or sets the time when the job state was last modified. + /// Gets or sets the time of the last meaningful change to the job: when it was started, when it was + /// stopped, when it was given a different from the one it already had, or + /// when it last recorded a non-zero number of purged blobs. /// + /// + /// This is not a liveness or heartbeat signal, and it must not be read as one. Starting a host does not + /// move it, and an active job whose cycles keep finding nothing to purge leaves it untouched indefinitely, + /// so a value far in the past is equally consistent with a healthy idle job and a dead one. + /// public DateTimeOffset? LastModifiedAt { get; set; } /// diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 8b61237f..37449725 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -110,6 +110,97 @@ public async Task Create_WhenAlreadyActive_SignalsNothingOtherThanRun() Times.Never); } + [Fact] + public async Task Create_WhenAlreadyActive_AndBatchSizeUnchanged_DoesNotMoveLastModifiedAt() + { + // Arrange - the steady state. Create runs on every host start, and almost every one of those carries + // the same configured batch size the job already has. If that rewrote LastModifiedAt, the field would + // degrade to "time of the last host start" and say nothing about the job. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 250); + + // Act + await this.job.RunAsync(operation); + + // Assert - the sibling test below is the positive control: identical wiring, differing only in the + // batch size passed in, and it proves this same path does move the field when something changes. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.LastModifiedAt.Should().Be(configuredAt); + state.PurgeBatchSize.Should().Be(250); + } + + [Fact] + public async Task Create_WhenAlreadyActive_AndBatchSizeChanged_MovesLastModifiedAt() + { + // Arrange - a real configuration change reaching an active job, which is the one thing this path + // exists to deliver and the one case that must be recorded. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 500); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.PurgeBatchSize.Should().Be(500); + state.LastModifiedAt.Should().BeAfter(configuredAt); + } + + [Fact] + public async Task Run_DoesNotMoveLastModifiedAt() + { + // Arrange - Run schedules an orchestrator and changes nothing about the job. It is signalled by every + // Create, so writing here would move the field on every host start and undo the conditional write + // above. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert - scheduling is asserted first as the positive control. Without it a misdispatched operation + // would write nothing and pass this test for entirely the wrong reason. + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.LastModifiedAt.Should().Be(configuredAt); + } + [Fact] public async Task Run_WhenActive_SchedulesOrchestratorAtTheFixedInstanceId() { @@ -206,9 +297,10 @@ public async Task Stop_WhenActive_MovesToPendingAndKeepsHistory() [Fact] public async Task Stop_WhenNotActive_LeavesStateUntouched() { - // Arrange - a job that is already stopped. Every host with auto-purge disabled signals Stop on each - // start, so the repeat is the common case; rewriting LastModifiedAt each time would destroy its only - // useful meaning, which is when the job actually stopped. + // Arrange - a job that is already stopped. The starter's client-side pre-check normally suppresses a + // redundant stop, so this is the signal that races past it: the job stopped between that read and this + // signal landing. Rewriting LastModifiedAt here would report the losing side of that race as if it + // were the moment the job stopped. DateTimeOffset stoppedAt = DateTimeOffset.UtcNow.AddHours(-6); BlobPurgeJobState existing = new() { From 2b389fa41683aa510cfc954d53d02d5cace06c8c Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 21:51:24 -0700 Subject: [PATCH 31/47] Drop an uncorrelatable backend column name from a proto comment The comment on LargePayloadPurgeResult.disposition told the reader to correlate a row with worker telemetry by (partitionId, instanceKey, payloadId) plus LastAttemptAt. The first three are fields on both messages. The fourth is a column on a backend table that is never sent to the worker - LargePayloadTombstone carries partitionId, instanceKey, payloadId, token and revision, and nothing else - so the advice named a value its audience can neither receive nor query, while also naming backend internals in a public repository. The remaining three identifiers are a complete correlation key for the telemetry this comment is about. The wire contract is unaffected: only a comment changed, and the serialized file descriptor embedded in the generated code is unchanged. The generated C# does move, because protoc copies proto comments into XML documentation, and that one documentation line now matches. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- src/Grpc/orchestrator_service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 51f490a2..080b697d 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -893,7 +893,7 @@ message LargePayloadPurgeResult { // The only field the backend acts on. Deliberately the only outcome field on this message: // anything finer would be write-only. Failure detail stays in the worker's own telemetry, which // holds the full exception rather than a lossy classification, and a row is correlated to it by - // (partitionId, instanceKey, payloadId) plus the ledger's LastAttemptAt. + // (partitionId, instanceKey, payloadId). LargePayloadPurgeDisposition disposition = 5; } From 3dd2b14cf26b29cbf7ebb6170358470b4d8502e0 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 22:23:40 -0700 Subject: [PATCH 32/47] Reconcile the blob auto-purge job periodically instead of once per host start The ensure loop returned as soon as one schedule succeeded, so it was a retry-until-success loop rather than a periodic one. That call is the only thing that re-signals the job's Run, so an orchestrator that died in the middle of a host's lifetime stayed down until the process was restarted. Every path now picks a delay and the loop waits once at its tail: the two success paths take a five minute reconcile interval, and the failure path keeps the short retry delay, because a backend that is unreachable at host start has to be retried quickly - until one pass succeeds the job may not exist at all. Nothing else changes. A pass that finds everything healthy is already a no-op at both hops it drives: the bridge's Create no-ops while the entity is Active, and the Run it signals is discarded by the backend while the orchestrator is alive. Two logs on the repeating path drop from Information to Debug, since they now fire once per host per interval forever. EventId 811 also said the batch size "was updated", which stopped being true when that write became conditional; it now says it was reconciled, which is true whether or not it changed. The comments that described recovery as bounded by host starts are rewritten, in the starter, in the entity and in the tests. The interval carries the cost it trades: this is per host, not per task hub, because staggered hosts each find the previous bridge already Completed and therefore replaceable, so a fleet of N pays roughly N bridge runs and N entity calls per interval. What keeps the purge work itself single is the fixed orchestrator instance id, not the bridge's dedupe policy. Tests cover that the loop repeats, and that shutdown cancels a pass parked in the interval rather than sitting it out - the latter being the one real regression risk, and one the other tests could only show as a hang. Both were confirmed to fail when the behaviour they name is removed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 79 ++++++++++++---- .../AutoPurge/Entity/BlobPurgeJob.cs | 17 ++-- .../AzureBlobPayloads/AutoPurge/Logs.cs | 4 +- .../AutoPurge/Models/BlobPurgeJobState.cs | 7 +- .../AutoPurge/BlobPurgeJobStarterTests.cs | 93 ++++++++++++++++++- .../AutoPurge/BlobPurgeJobTests.cs | 14 +-- 6 files changed, 174 insertions(+), 40 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index d43e47c5..af197a99 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -16,7 +16,8 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// resolved options. It is registered unconditionally by UseExternalizedPayloads and decides what to do at /// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, stops a /// running job when it is disabled, and no-ops with an error log when the registered store cannot delete. It -/// never blocks host startup - the work runs on a background task that retries until the backend is reachable. +/// never blocks host startup - the work runs on a background task that retries until the backend is reachable +/// and then, on the enabled path, keeps reconciling on a fixed interval for the lifetime of the process. /// The job is a per-task-hub singleton, so racing client processes converge on the same result. /// sealed class BlobPurgeJobStarter : IHostedService, IDisposable @@ -55,6 +56,33 @@ public BlobPurgeJobStarter( this.logger = Check.NotNull(logger); } + /// + /// Gets or sets the interval between reconciliation passes on the enabled path. Settable only so tests can + /// drive the loop without waiting on wall-clock time; nothing in the product ever assigns it. + /// + /// + /// + /// This value is the worst-case time a dead job stays dead. Nothing else re-signals the job's orchestrator, + /// so recovery cannot be faster than one pass. What it buys that back with is one schedule call per host + /// per interval, forever - so it trades a bounded recovery time against a standing, fleet-sized cost, and + /// shortening it makes that cost grow in proportion. + /// + /// + /// That cost is per host, not per task hub. Hosts start at different times, so their intervals are + /// staggered, and a pass almost always finds the previous bridge already Completed - which this call leaves + /// replaceable on purpose - so it schedules and runs a bridge of its own. The + /// path only absorbs passes that overlap a bridge which is + /// still Pending or Running, and that window is narrow because a bridge makes one entity call and exits. So + /// a fleet of N hosts costs roughly N bridge runs and N entity calls per interval, not one. + /// + /// + /// What keeps the actual purge work single is not that dedupe policy but the fixed orchestrator instance id: + /// every bridge run signals Run, and the backend discards a start aimed at an orchestrator that is still + /// alive. One perpetual orchestrator therefore serves the whole fleet however many bridges ran. + /// + /// + internal TimeSpan ReconcileInterval { get; set; } = TimeSpan.FromMinutes(5); + /// public Task StartAsync(CancellationToken cancellationToken) { @@ -102,7 +130,7 @@ public Task StartAsync(CancellationToken cancellationToken) int batchSize = opts.PayloadPurgeBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend - // is reachable. + // is reachable, and keep reconciling it from there. this.cts = new CancellationTokenSource(); this.backgroundTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); return Task.CompletedTask; @@ -141,6 +169,8 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT { while (!cancellationToken.IsCancellationRequested) { + TimeSpan delay; + try { BlobPurgeJobOperationRequest request = new( @@ -163,13 +193,13 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // // Re-running a bridge that already finished is safe and close to free: the bridge's only effect // is calling Create, which no-ops while the entity is Active, so the cost is one instance - // replacement plus one entity call per host start. + // replacement plus one entity call per reconciliation pass. // // Re-running is also what lets the job self-heal after the entity is removed, for example by // CleanEntityStorageAsync. The perpetual orchestrator exits cleanly when it reads back a null // entity state, and a removed entity is back to its default Pending status, so the job is left // dead with a Completed bridge behind it. Keeping Completed deduped would keep it dead - // permanently; making it replaceable means the next host start re-runs Create, which finds the + // permanently; making it replaceable means the next pass re-runs Create, which finds the // entity not Active and rebuilds the job. // // This also recovers the case where the perpetual orchestrator dies while the entity is still @@ -179,8 +209,13 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // So a healthy job is left untouched and a dead one is rebuilt, without this side ever having to // ask which of the two it is looking at. // - // Recovery is bounded by host starts rather than being continuous: nothing re-signals Run - // between them, so an orchestrator that dies mid-lifetime stays down until the next Create. + // Recovery repeats on a fixed interval rather than being bounded by host starts. This loop runs + // for the lifetime of the process and re-issues the same call every ReconcileInterval, so an + // orchestrator that dies mid-lifetime is rebuilt within roughly one interval instead of staying + // down until the next deployment. Repeating it costs nothing extra to reason about, because a + // pass that finds everything healthy is already a no-op at both hops: the bridge's Create + // no-ops while the entity is Active, and the Run it signals is discarded by the backend while + // the orchestrator is alive. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, @@ -191,7 +226,7 @@ await client.ScheduleNewOrchestrationInstanceAsync( cancellationToken); this.logger.BlobPurgeJobEnsured(); - return; + delay = this.ReconcileInterval; } catch (OrchestrationAlreadyExistsException) { @@ -199,12 +234,12 @@ await client.ScheduleNewOrchestrationInstanceAsync( // policy it means another host scheduled the bridge and it is still Pending or Running. That is // exactly the concurrent-start race this replaced a status check to close: one create wins and // the loser lands here. Either way the singleton is already being set up, so treat it as - // ensured and stop. + // ensured and wait for the next pass. // // Note this is NOT the steady-state path. A bridge that already finished is Completed, which is - // replaceable, so a later host start re-runs it rather than landing here. + // replaceable, so a later pass re-runs it rather than landing here. this.logger.BlobPurgeJobEnsured(); - return; + delay = this.ReconcileInterval; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -212,15 +247,23 @@ await client.ScheduleNewOrchestrationInstanceAsync( } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { + // Deliberately the short delay and not the reconcile interval. A backend that is unreachable + // when the host starts has to be retried quickly, because until one pass succeeds the job may + // not exist at all; the long interval is only the price of keeping a job that already exists + // healthy. this.logger.BlobPurgeStarterRetry(ex); - try - { - await Task.Delay(RetryDelay, cancellationToken); - } - catch (OperationCanceledException) - { - return; - } + delay = RetryDelay; + } + + try + { + await Task.Delay(delay, cancellationToken); + } + catch (OperationCanceledException) + { + // Shutdown. This is the only thing that ends the loop on the success path, which is why + // StopAsync can cancel a pass that is parked here instead of waiting out the interval. + return; } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index ce2199f7..f1b704e7 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -32,10 +32,10 @@ public void Create(TaskEntityContext context, int purgeBatchSize) // rejects would wedge it permanently. // // Written only when the value actually differs, which is what keeps LastModifiedAt tracking real - // configuration changes. Create runs on every host start, so an unconditional write would reduce - // the field to "time of the last host start". An entity written by a build that predates this - // field carries zero, which differs from any configured size, so the first Create after an upgrade - // still repairs it. + // configuration changes. Create runs on every reconciliation pass, not only at host start, so an + // unconditional write would reduce the field to "time of the last pass". An entity written by a + // build that predates this field carries zero, which differs from any configured size, so the + // first Create after an upgrade still repairs it. if (this.State.PurgeBatchSize != purgeBatchSize) { this.State.PurgeBatchSize = purgeBatchSize; @@ -45,8 +45,9 @@ public void Create(TaskEntityContext context, int purgeBatchSize) logger.BlobPurgeJobAlreadyRunning(context.Id.Key); // Run is re-signalled even though the job is already active, and this is what makes the job - // self-heal: Create runs on every host start, so a job whose orchestrator has died is rebuilt at the - // next one. The signal is deliberately blind. An entity-initiated start carries no reuse policy, so + // self-heal: Create runs on every reconciliation pass, so a job whose orchestrator has died is + // rebuilt within roughly one interval rather than waiting for the next host start. The signal is + // deliberately blind. An entity-initiated start carries no reuse policy, so // the backend decides its fate: it discards the start while the target instance exists in any // non-completed status, and purges and replaces it once the instance has completed, terminated, // failed or been canceled. A healthy orchestrator is therefore left strictly alone and only a dead @@ -86,8 +87,8 @@ public void Create(TaskEntityContext context, int purgeBatchSize) /// /// /// This operation deliberately writes no state. It schedules an orchestrator and nothing more, and it runs - /// on every host start, so touching here would overwrite a - /// real change with the time of a start that changed nothing. + /// on every reconciliation pass, so touching here would + /// overwrite a real change with the time of a pass that changed nothing. /// /// /// The entity context. diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index b4042edd..3f3ae217 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -13,7 +13,7 @@ static partial class Logs [LoggerMessage(EventId = 810, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' created.")] public static partial void BlobPurgeJobCreated(this ILogger logger, string? jobId); - [LoggerMessage(EventId = 811, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' is already active. Its batch size was updated from the current configuration, and its orchestrator was re-signalled, which starts one only if none is running.")] + [LoggerMessage(EventId = 811, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is already active. Its batch size was reconciled against the current configuration, and its orchestrator was re-signalled, which starts one only if none is running.")] public static partial void BlobPurgeJobAlreadyRunning(this ILogger logger, string? jobId); [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] @@ -31,7 +31,7 @@ static partial class Logs [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' stopped. The perpetual orchestrator is not terminated; it reads the job state at the start of its next cycle and exits on its own.")] public static partial void BlobPurgeJobStopped(this ILogger logger, string? jobId); - [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] + [LoggerMessage(EventId = 817, Level = LogLevel.Debug, Message = "Blob payload auto-purge singleton job ensured.")] public static partial void BlobPurgeJobEnsured(this ILogger logger); [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not reach the singleton job; retrying.")] diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs index 0eee8dd2..0a17caae 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs @@ -24,9 +24,10 @@ public sealed class BlobPurgeJobState /// when it last recorded a non-zero number of purged blobs. /// /// - /// This is not a liveness or heartbeat signal, and it must not be read as one. Starting a host does not - /// move it, and an active job whose cycles keep finding nothing to purge leaves it untouched indefinitely, - /// so a value far in the past is equally consistent with a healthy idle job and a dead one. + /// This is not a liveness or heartbeat signal, and it must not be read as one. Neither starting a host nor + /// a reconciliation pass moves it, and an active job whose cycles keep finding nothing to purge leaves it + /// untouched indefinitely, so a value far in the past is equally consistent with a healthy idle job and a + /// dead one. /// public DateTimeOffset? LastModifiedAt { get; set; } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index a6a3db91..83d51fb2 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -212,11 +212,82 @@ public async Task EnsureJob_SchedulesBridge_DedupingOnlyPendingAndRunning() options!.InstanceId.Should().Be(BlobPurgeConstants.StarterInstanceId); // Every status other than Pending and Running is replaceable, so a finished bridge is re-run on the - // next host start. That is what lets the job rebuild itself after the entity is removed. The set is - // asserted exactly, never as a superset, because the hazard is a silent omission. + // next reconciliation pass. That is what lets the job rebuild itself after the entity is removed. The + // set is asserted exactly, never as a superset, because the hazard is a silent omission. options.DedupeStatuses.Should().BeEquivalentTo(["Pending", "Running"]); } + [Fact] + public async Task EnsureJob_AfterSuccessfulSchedule_KeepsReconciling() + { + // Arrange - the loop used to return as soon as one schedule succeeded, which left recovery bounded by + // host starts: this call is the only thing that re-signals the job's Run, so an orchestrator that died + // mid-lifetime stayed down until the process was restarted. It has to keep re-issuing the call instead. + Mock client = new("test"); + TaskCompletionSource secondSchedule = new(); + int schedules = 0; + client + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(() => + { + if (Interlocked.Increment(ref schedules) >= 2) + { + secondSchedule.TrySetResult(true); + } + }) + .ReturnsAsync(BlobPurgeConstants.StarterInstanceId); + + // A short interval rather than a fake clock. The test still gates on the second call happening, never + // on time having passed, so it neither sleeps nor assumes anything about how long a pass takes. + BlobPurgeJobStarter starter = EnabledStarterFor(client.Object); + starter.ReconcileInterval = TimeSpan.FromMilliseconds(10); + + // Act + await starter.StartAsync(CancellationToken.None); + Task completed = await Task.WhenAny(secondSchedule.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await starter.StopAsync(CancellationToken.None); + + // Assert - a loop that returned after its first success leaves this waiting until the timeout fires. + completed.Should().BeSameAs( + secondSchedule.Task, "reconciliation must repeat rather than ending after the first success"); + } + + [Fact] + public async Task StopAsync_WhileWaitingForNextReconcile_ReturnsPromptly() + { + // Arrange - the default interval, so the loop parks in a five-minute wait once its first pass succeeds. + // StopAsync awaits the background task, so a wait that did not observe cancellation would hold host + // shutdown for the rest of the interval. That is the one real regression risk in making the loop + // periodic, and it is invisible in the other tests because they would simply hang. + Mock client = new("test"); + TaskCompletionSource scheduled = new(); + client + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(() => scheduled.TrySetResult(true)) + .ReturnsAsync(BlobPurgeConstants.StarterInstanceId); + + BlobPurgeJobStarter starter = EnabledStarterFor(client.Object); + + // Act - waiting for the first schedule is the positive control. Without it a prompt StopAsync would + // also be what a starter that never reached the loop at all produces. + await starter.StartAsync(CancellationToken.None); + Task firstPass = await Task.WhenAny(scheduled.Task, Task.Delay(TimeSpan.FromSeconds(30))); + Task stop = starter.StopAsync(CancellationToken.None); + Task stopped = await Task.WhenAny(stop, Task.Delay(TimeSpan.FromSeconds(30))); + + // Assert + firstPass.Should().BeSameAs(scheduled.Task, "the loop must have reached the wait being cancelled here"); + stopped.Should().BeSameAs(stop, "shutdown must cancel the wait rather than sit out the interval"); + } + static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) { Mock> monitor = new(); @@ -224,6 +295,24 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag return monitor.Object; } + /// + /// Builds a starter with auto-purge enabled over the given client. The real blob store is used because the + /// store-capability gate refuses to start the job for any other kind, so a stub would never reach the + /// ensure loop. UseDevelopmentStorage=true constructs it offline, with no network I/O. + /// + static BlobPurgeJobStarter EnabledStarterFor(DurableTaskClient client) + { + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(client); + + return new BlobPurgeJobStarter( + provider.Object, + new BlobPayloadStore(new LargePayloadStorageOptions("UseDevelopmentStorage=true")), + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = true }), + "test", + new TestLogger()); + } + /// /// Builds entity metadata carrying a job in the given status. /// diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 37449725..88d4fa6a 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -70,8 +70,8 @@ public async Task Create_WhenAlreadyActive_UpdatesBatchSizeAndReSignalsRun() state.PurgeBatchSize.Should().Be(999); // Run is re-signalled even though the job is already active. That is what lets a job whose orchestrator - // has died be rebuilt at the next host start, and it is safe because the backend discards the resulting - // start while the orchestrator is alive rather than replacing it. + // has died be rebuilt at the next reconciliation pass, and it is safe because the backend discards the + // resulting start while the orchestrator is alive rather than replacing it. Mock.Get(operation.Context).Verify( c => c.SignalEntity( It.IsAny(), @@ -113,9 +113,9 @@ public async Task Create_WhenAlreadyActive_SignalsNothingOtherThanRun() [Fact] public async Task Create_WhenAlreadyActive_AndBatchSizeUnchanged_DoesNotMoveLastModifiedAt() { - // Arrange - the steady state. Create runs on every host start, and almost every one of those carries - // the same configured batch size the job already has. If that rewrote LastModifiedAt, the field would - // degrade to "time of the last host start" and say nothing about the job. + // Arrange - the steady state. Create runs on every reconciliation pass, and almost every one of those + // carries the same configured batch size the job already has. If that rewrote LastModifiedAt, the field + // would degrade to "time of the last pass" and say nothing about the job. DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); BlobPurgeJobState existing = new() { @@ -170,8 +170,8 @@ public async Task Create_WhenAlreadyActive_AndBatchSizeChanged_MovesLastModified public async Task Run_DoesNotMoveLastModifiedAt() { // Arrange - Run schedules an orchestrator and changes nothing about the job. It is signalled by every - // Create, so writing here would move the field on every host start and undo the conditional write - // above. + // Create, so writing here would move the field on every reconciliation pass and undo the conditional + // write above. DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); BlobPurgeJobState existing = new() { From 3a1b673b35bfb6fb600df11d9709c6b5ea1bb41d Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 13 Aug 2026 10:22:25 -0700 Subject: [PATCH 33/47] Correct comments that claimed the purge starter defers client construction Two comments in BlobPurgeJobStarter said the client is resolved "lazily" so that a DurableTaskClient is not constructed at host start for apps that externalize payloads without auto-purge. That is not what happens. DefaultDurableTaskClientProvider takes IEnumerable, which the DI container materializes eagerly, and each ClientContainer is built by running the client builder. The provider is constructed before any hosted service's StartAsync runs, so every client already exists by the time either path calls GetClient - GetClient is only a FirstOrDefault lookup and constructs nothing. Deferring that call defers no construction. What is true, and is kept, is the real asymmetry between the two paths. The enabled path resolves in StartAsync's body, so a builder name matching no client throws ArgumentOutOfRangeException straight out of host startup; placing it after the AutoPurge and store gates keeps apps that never enabled auto-purge from reaching it. The disabled path resolves inside the stop loop's try, where the same failure is caught, logged and retried rather than faulting the background task. The "resolved yet" wording is dropped: the client set is fixed when the provider is constructed, so a name that misses once misses always. Also documents why both background loops are launched with Task.Run and CancellationToken.None: an async method runs synchronously until its first suspending await, and these loops' awaits land on client implementations this code does not own, so Task.Run makes "StartAsync returns immediately" a local property; CancellationToken.None keeps a fast StopAsync from dropping the stop signal before it is sent. Comments only; no behavioural change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index af197a99..97807479 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -106,6 +106,9 @@ public Task StartAsync(CancellationToken cancellationToken) // // The job's state is read before anything is signalled, so an app that never enabled auto-purge // signals nothing and no entity is created for it. + // + // Launched on a background task for the reasons given on the enabled path below; CancellationToken.None + // matters most here, because dropping this delegate would leave a running job's stop signal unsent. this.cts = new CancellationTokenSource(); this.backgroundTask = Task.Run(() => this.SignalJobStopAsync(this.cts.Token), CancellationToken.None); return Task.CompletedTask; @@ -123,14 +126,24 @@ public Task StartAsync(CancellationToken cancellationToken) } // Resolve the client by builder name rather than by type: a named client builder must get its own - // client, and resolving lazily here - after the AutoPurge gate - avoids constructing a DurableTaskClient - // at host start for apps that externalize payloads without auto-purge. + // client. This resolve sits after the AutoPurge and store gates because it runs directly in StartAsync's + // body, so a builder name that matches no registered client throws ArgumentOutOfRangeException straight + // out of host startup; keeping it behind the gates means an app that has not enabled auto-purge, or whose + // store cannot delete, never reaches it. It does not defer client construction: the provider is handed an + // already-materialized set of clients when it is constructed, before any StartAsync runs. The disabled + // path instead resolves inside its loop, so a bad name cannot throw out of startup; see SignalJobStopAsync. DurableTaskClient client = this.clientProvider.GetClient(this.builderName); int batchSize = opts.PayloadPurgeBatchSize; - // Do not block host startup; ensure the job on a background task with basic retry until the backend - // is reachable, and keep reconciling it from there. + // EnsureJobAsync is a perpetual loop, and an async method runs synchronously on its caller's thread + // until the first await that actually suspends. Launching it directly would make "StartAsync returns" + // hinge on some await inside the loop genuinely suspending - awaits that land on DurableTaskClient and + // DurableEntityClient implementations this code does not own. Task.Run makes "StartAsync returns + // immediately" a local, visible property instead, for one allocation per process. CancellationToken.None + // is deliberate: that argument governs only whether the delegate is invoked, so passing this.cts.Token + // would let a fast StopAsync drop the delegate before it runs; the loop cancels through the token passed + // into the method instead. this.cts = new CancellationTokenSource(); this.backgroundTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); return Task.CompletedTask; @@ -274,11 +287,13 @@ async Task SignalJobStopAsync(CancellationToken cancellationToken) { try { - // Resolved inside the loop rather than on the host-start path. This path is taken by every app - // that externalizes payloads without auto-purge, and constructing a DurableTaskClient at host - // start for them is exactly what the enable path's lazy resolution avoids. It also means a - // client that cannot be resolved yet is retried here instead of throwing out of host startup, - // which must never happen for a feature the user has turned off. + // Resolved inside this loop's try, not in StartAsync's body. A builder name that matches no + // registered client makes GetClient throw; catching it here - and retrying after a delay - keeps + // that failure from surfacing as an unobserved fault for a feature the user has turned off. The + // enabled path is genuinely different: its resolve sits directly in StartAsync's body, so the same + // bad name would throw straight out of host startup. The client set is fixed when the provider is + // constructed, so a name that misses once misses always; the retry cannot make it resolve, it only + // keeps the failure contained. DurableTaskClient client = this.clientProvider.GetClient(this.builderName); // Held in a local so the query and the signal below are issued against the same object. Both go From 4cc0baacd80f3c56758038e2b85925346ff5ee27 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 14 Aug 2026 09:53:47 -0700 Subject: [PATCH 34/47] Enable entity support when externalized payloads are configured The auto-purge starter reaches its singleton BlobPurgeJob entity through client.Entities on both paths (enabled: create/drive the job; disabled: signal Stop), but nothing in the AzureBlobPayloads extension turned entity support on. With EnableEntitySupport defaulting to false, the disabled path (AutoPurge == false, the default) threw NotSupportedException from client.Entities, was caught by the starter's outer retry, and looped forever - logging an error every 10 seconds for the life of the process. Set EnableEntitySupport = true in both UseExternalizedPayloadsCore methods, on the base client and worker options (not gated on AutoPurge, which would leave the broken path in place and break the off-switch). On the client this is applied to the base options so an explicit UseGrpc override still wins via DurableTaskClientOptions.ApplyTo. Each side mirrors its own AzureManaged precedent. Adds client and worker regression tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- ...ientBuilderExtensions.AzureBlobPayloads.cs | 13 ++++++ ...rkerBuilderExtensions.AzureBlobPayloads.cs | 11 +++++ .../UseExternalizedPayloadsTests.cs | 42 +++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index f54d499b..ed172b0f 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -87,6 +87,19 @@ static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientB } }); + // The auto-purge job is entity-driven: BlobPurgeJobStarter reaches the singleton job through + // client.Entities on BOTH paths - the enabled path creates and drives the job, and the disabled path + // signals it Stop - so entity support must be on whenever externalized payloads are configured, not only + // when AutoPurge is enabled. Gating it on AutoPurge would break the off-switch: flipping AutoPurge from + // true to false would disable entities exactly when the stop signal needs client.Entities, leaving a + // running job deleting blobs forever. Set it on the base options so an explicit UseGrpc client that + // disables entity support still wins (DurableTaskClientOptions.ApplyTo copies this value only when the + // derived options did not set it explicitly). + builder.Configure(options => + { + options.EnableEntitySupport = true; + }); + // Always register the auto-purge starter. Whether auto-purge is actually enabled can only be known once // options are fully resolved - the flag can be set by the inline configure delegate, services.Configure, // configuration binding or PostConfigure, none of which are visible here at registration time - so the diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index 142f96eb..ebe452fb 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -91,6 +91,17 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB opt.LargePayloadAutoPurgeEnabled = opts.AutoPurge; }); + // The auto-purge job is entity-driven: its orchestrator drives the BlobPurgeJob entity, and an + // orchestrator that touches entities with support off throws (TaskOrchestrationContextWrapper). Enable + // it whenever externalized payloads are configured - mirroring the client side, and not gated on + // AutoPurge - so the job can be both driven (enabled path) and stopped (disabled path) either way. + builder.Services + .AddOptions(builder.Name) + .Configure(options => + { + options.EnableEntitySupport = true; + }); + // Register the entity/orchestrators/activities that run the singleton auto-purge job. These are // ALWAYS registered (not gated on AutoPurge) so that a client-enabled job always has something to // execute here. The purge activities fetch/report via the injected DurableTaskClient. diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index 7ab88c15..f0e59160 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -154,4 +154,46 @@ public void UseExternalizedPayloads_Worker_SendsResolvedAutoPurgeValueOnTheHands // Assert - an explicit true/false, never left absent, because calling this extension IS the choice. grpcOptions.LargePayloadAutoPurgeEnabled.Should().Be(autoPurge); } + + [Fact] + public void UseExternalizedPayloads_Client_EnablesEntitySupport() + { + // Arrange - the auto-purge starter reaches the singleton job through client.Entities on BOTH the enabled + // and disabled paths, so the client must turn entity support on whenever externalized payloads are + // configured. It must not be gated on AutoPurge, so leave AutoPurge at its default (false) here. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act + builder.Object.UseExternalizedPayloads(options => { }); + using ServiceProvider provider = services.BuildServiceProvider(); + DurableTaskClientOptions options = + provider.GetRequiredService>().Get(string.Empty); + + // Assert + options.EnableEntitySupport.Should().BeTrue(); + } + + [Fact] + public void UseExternalizedPayloads_Worker_EnablesEntitySupport() + { + // Arrange - the purge orchestrator drives the BlobPurgeJob entity, and an orchestrator that touches + // entities with support off throws, so the worker must turn entity support on whenever externalized + // payloads are configured. Not gated on AutoPurge, so leave AutoPurge at its default (false) here. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act + builder.Object.UseExternalizedPayloads(options => { }); + using ServiceProvider provider = services.BuildServiceProvider(); + DurableTaskWorkerOptions options = + provider.GetRequiredService>().Get(string.Empty); + + // Assert + options.EnableEntitySupport.Should().BeTrue(); + } } From b8e581b1db7966f487ec98292ce6712ae4da88fc Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 14 Aug 2026 21:15:23 -0400 Subject: [PATCH 35/47] Purge externalized blobs over the worker's own gRPC transport The two auto-purge activities injected a concrete DurableTaskClient, but AddDurableTaskWorker never registers one - only AddDurableTaskClient does. In a worker-only ("client triggers, worker executes") split deployment with auto-purge enabled, resolving either activity threw at dispatch time - and only at dispatch, because DurableTaskRegistry stores a lazy ActivatorUtilities.GetServiceOrCreateInstance factory - so auto-purge was silently 100% broken and blobs accumulated forever. Introduce an internal ILargePayloadPurgeClient over exactly the two backend RPCs the job needs (fetch tombstones, report results) plus a gRPC implementation that reuses the worker's OWN transport: it builds a TaskHubSidecarServiceClient from the named GrpcDurableTaskWorkerOptions, mirroring the worker's Channel -> CallInvoker precedence, and resolves it lazily on first RPC so the activities stay constructible in hosts that never run the job. The marshalling (limit 1..1000 validation, empty short-circuit, RpcException/Cancelled -> OperationCanceledException, by-value disposition mapping) is copied verbatim from GrpcDurableTaskClient so the wire behavior is identical; its public virtuals and overrides are left untouched. Both purge activities now inject ILargePayloadPurgeClient instead of DurableTaskClient and become internal sealed (no new public API surface; the test project already sees internals via the auto-generated InternalsVisibleTo). Registers the client in the worker's UseExternalizedPayloadsCore against builder.Name. Adds a worker-only regression test asserting both activities construct via the same reflection path the registry uses at dispatch. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- .../GetLargePayloadTombstonesActivity.cs | 8 +- .../ReportLargePayloadPurgeResultsActivity.cs | 8 +- .../Client/GrpcLargePayloadPurgeClient.cs | 157 ++++++++++++++++++ .../Client/ILargePayloadPurgeClient.cs | 35 ++++ ...rkerBuilderExtensions.AzureBlobPayloads.cs | 13 +- .../UseExternalizedPayloadsTests.cs | 27 +++ 6 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Client/GrpcLargePayloadPurgeClient.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Client/ILargePayloadPurgeClient.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs index 273989ea..3be72bc2 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs @@ -10,15 +10,15 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// Activity that fetches a bounded batch of due large-payload tombstones from the backend for the auto-purge /// job to delete. /// -/// The Durable Task client used to query the backend for tombstones. +/// The purge client used to query the backend for tombstones. /// The logger instance. [DurableTask] -public class GetLargePayloadTombstonesActivity( - DurableTaskClient client, +internal sealed class GetLargePayloadTombstonesActivity( + ILargePayloadPurgeClient client, ILogger logger) : TaskActivity> { - readonly DurableTaskClient client = Check.NotNull(client); + readonly ILargePayloadPurgeClient client = Check.NotNull(client); readonly ILogger logger = Check.NotNull(logger); /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs index 7003b7c9..e65f88e9 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs @@ -12,15 +12,15 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// reported, not only the successful ones: the backend owns retry scheduling, so a row it hears nothing about /// would simply be re-served unchanged on the next cycle. /// -/// The Durable Task client used to report purge results to the backend. +/// The purge client used to report purge results to the backend. /// The logger instance. [DurableTask] -public class ReportLargePayloadPurgeResultsActivity( - DurableTaskClient client, +internal sealed class ReportLargePayloadPurgeResultsActivity( + ILargePayloadPurgeClient client, ILogger logger) : TaskActivity, object?> { - readonly DurableTaskClient client = Check.NotNull(client); + readonly ILargePayloadPurgeClient client = Check.NotNull(client); readonly ILogger logger = Check.NotNull(logger); /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/GrpcLargePayloadPurgeClient.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/GrpcLargePayloadPurgeClient.cs new file mode 100644 index 00000000..df5d9c08 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/GrpcLargePayloadPurgeClient.cs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Worker.Grpc; +using Microsoft.Extensions.Options; +using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// gRPC implementation of that reuses the worker's own transport. It +/// builds a from the named , +/// so the two purge RPCs travel the exact same channel the worker already uses to reach the backend, with no +/// dependency on a client-side . The marshalling mirrors +/// GrpcDurableTaskClient's equivalent overrides so the wire behavior is identical either way. +/// +internal sealed class GrpcLargePayloadPurgeClient : ILargePayloadPurgeClient +{ + readonly IOptionsMonitor options; + readonly string name; + readonly object gate = new(); + TaskHubSidecarServiceClient? sidecarClient; + + /// + /// Initializes a new instance of the class. + /// + /// The monitor used to resolve the worker's gRPC options by builder name. + /// The builder name whose worker options carry the transport to reuse. + public GrpcLargePayloadPurgeClient(IOptionsMonitor options, string name) + { + this.options = Check.NotNull(options); + this.name = name ?? string.Empty; + } + + /// + public async Task> GetLargePayloadTombstonesAsync( + int limit, CancellationToken cancellation = default) + { + if (limit <= 0 || limit > 1000) + { + throw new ArgumentOutOfRangeException( + nameof(limit), limit, "Limit must be greater than 0 and less than or equal to 1000."); + } + + TaskHubSidecarServiceClient client = this.GetClient(); + + P.GetLargePayloadTombstonesResponse response; + try + { + response = await client.GetLargePayloadTombstonesAsync( + new P.GetLargePayloadTombstonesRequest { Limit = limit }, + cancellationToken: cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.GetLargePayloadTombstonesAsync)} operation was canceled.", e, cancellation); + } + + List result = new(response.Tombstones.Count); + foreach (P.LargePayloadTombstone tombstone in response.Tombstones) + { + result.Add(new LargePayloadTombstone( + tombstone.PartitionId, + tombstone.InstanceKey, + tombstone.PayloadId, + tombstone.Token, + tombstone.Revision)); + } + + return result; + } + + /// + public async Task ReportLargePayloadPurgeResultsAsync( + IEnumerable results, CancellationToken cancellation = default) + { + Check.NotNull(results); + + P.ReportLargePayloadPurgeResultsRequest request = new(); + foreach (LargePayloadPurgeResult result in results) + { + request.Results.Add(new P.LargePayloadPurgeResult + { + PartitionId = result.PartitionId, + InstanceKey = result.InstanceKey, + PayloadId = result.PayloadId, + Revision = result.Revision, + + // The managed disposition enum declares the same numeric values as its protobuf counterpart, + // so it maps across by value. This is the only enum on the message and it only travels + // outbound, so the SDK can never receive a value it does not know. + Disposition = (P.LargePayloadPurgeDisposition)result.Disposition, + }); + } + + if (request.Results.Count == 0) + { + return; + } + + TaskHubSidecarServiceClient client = this.GetClient(); + try + { + await client.ReportLargePayloadPurgeResultsAsync(request, cancellationToken: cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.ReportLargePayloadPurgeResultsAsync)} operation was canceled.", e, cancellation); + } + } + + TaskHubSidecarServiceClient GetClient() + { + // Resolve the transport lazily and cache it. The worker's Channel/CallInvoker is only guaranteed to be + // populated once the options have been fully post-configured (the AzureBlobPayloads worker setup moves + // the Channel onto CallInvoker, or throws if neither was set), which happens after this service is + // constructed. Building on first RPC keeps the purge activities constructible in hosts that never run + // the job - which is exactly the worker-only case this abstraction exists to unblock. + lock (this.gate) + { + if (this.sidecarClient is not null) + { + return this.sidecarClient; + } + + GrpcDurableTaskWorkerOptions opts = this.options.Get(this.name); + + // Mirror the worker's own Channel -> CallInvoker precedence (GrpcDurableTaskWorker.GetCallInvoker): + // reuse whichever the worker resolved so the purge RPCs share its exact transport. The worker also + // has an Address-only branch that builds an owned channel, but it is unreachable here - the + // AzureBlobPayloads worker PostConfigure guarantees one of these two is set or throws first - so we + // do not duplicate its per-target-framework channel-lifetime logic. + CallInvoker callInvoker; + if (opts.Channel is { } channel) + { + callInvoker = channel.CreateCallInvoker(); + } + else if (opts.CallInvoker is { } invoker) + { + callInvoker = invoker; + } + else + { + throw new InvalidOperationException( + "A gRPC Channel or CallInvoker must be configured on the worker to purge externalized payloads."); + } + + this.sidecarClient = new TaskHubSidecarServiceClient(callInvoker); + return this.sidecarClient; + } + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/ILargePayloadPurgeClient.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/ILargePayloadPurgeClient.cs new file mode 100644 index 00000000..789ed911 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/ILargePayloadPurgeClient.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Abstraction over the two backend RPCs the auto-purge job needs: fetching a batch of due large-payload +/// tombstones and reporting the outcome of each attempted blob deletion. The worker talks to the backend over +/// its own gRPC transport through this abstraction, so the purge activities never depend on a +/// - which a worker-only ("client triggers, worker executes") host would never +/// register. +/// +internal interface ILargePayloadPurgeClient +{ + /// + /// Fetches a bounded batch of due large-payload tombstones from the backend. + /// + /// The maximum number of tombstones to return. Must be in the range 1..1000. + /// The cancellation token. + /// The due tombstones, up to of them. + Task> GetLargePayloadTombstonesAsync( + int limit, CancellationToken cancellation = default); + + /// + /// Reports the outcome of every attempted blob deletion to the backend so it can hard-delete the resolved + /// rows, reschedule the retryable ones, and quarantine the rest. + /// + /// The per-tombstone purge results to report. + /// The cancellation token. + /// A task that completes once the backend has accepted the report. + Task ReportLargePayloadPurgeResultsAsync( + IEnumerable results, CancellationToken cancellation = default); +} diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index ebe452fb..5606b864 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -102,9 +102,20 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB options.EnableEntitySupport = true; }); + // The purge activities fetch tombstones and report results over the worker's OWN gRPC transport, so a + // worker-only host (which never registers a DurableTaskClient) can still run the job. Resolve the + // transport from the same named GrpcDurableTaskWorkerOptions the worker uses, capturing builder.Name so a + // named worker binds to its own options. + builder.Services.TryAddSingleton(sp => + { + IOptionsMonitor options = + sp.GetRequiredService>(); + return new GrpcLargePayloadPurgeClient(options, builder.Name); + }); + // Register the entity/orchestrators/activities that run the singleton auto-purge job. These are // ALWAYS registered (not gated on AutoPurge) so that a client-enabled job always has something to - // execute here. The purge activities fetch/report via the injected DurableTaskClient. + // execute here. The purge activities fetch/report via the injected ILargePayloadPurgeClient above. builder.AddTasks(r => { r.AddEntity(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index f0e59160..1ad2627a 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -196,4 +196,31 @@ public void UseExternalizedPayloads_Worker_EnablesEntitySupport() // Assert options.EnableEntitySupport.Should().BeTrue(); } + + [Fact] + public void UseExternalizedPayloads_WorkerOnly_PurgeActivitiesAreConstructible() + { + // Arrange - a WORKER-ONLY host: AddDurableTaskWorker + UseExternalizedPayloads, with no AddDurableTaskClient + // anywhere. This is the split-deployment shape ("client triggers, worker executes"). The purge activities + // used to inject a concrete DurableTaskClient, which a worker-only host never registers, so they threw at + // dispatch time - and only at dispatch, because DurableTaskRegistry stores a lazy + // ActivatorUtilities.GetServiceOrCreateInstance factory - leaving auto-purge silently broken. They now + // inject the worker's own ILargePayloadPurgeClient, so they construct with no DurableTaskClient present. + ServiceCollection services = new(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddDurableTaskWorker( + builder => builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true")); + + using ServiceProvider provider = services.BuildServiceProvider(); + + // Act - construct both activities through the exact reflection path DurableTaskRegistry uses at dispatch. + Action constructGet = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(GetLargePayloadTombstonesActivity)); + Action constructReport = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(ReportLargePayloadPurgeResultsActivity)); + + // Assert - both must resolve without a client in the container. This throws on the pre-fix code. + constructGet.Should().NotThrow(); + constructReport.Should().NotThrow(); + } } From 51a0f1c5ee05d8cf212c69b73c64033f68e61595 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 14 Aug 2026 23:37:17 -0400 Subject: [PATCH 36/47] Collapse purge client abstraction into direct sidecar client injection ILargePayloadPurgeClient had exactly one implementation and no mocks, and GrpcLargePayloadPurgeClient's lazy GetClient()/lock/cache machinery merely duplicated what DI already does: a TryAddSingleton factory is already lazy (it runs on first resolve, which is at dispatch) and DI already caches the singleton. Register the worker's TaskHubSidecarServiceClient directly from the named GrpcDurableTaskWorkerOptions (CallInvoker, then Channel) and inline the tombstone/report marshalling into the two activities, which now inject the sidecar client instead of the removed abstraction. Behavior is unchanged: the RPCs still ride the worker's own transport, so a worker-only host runs the purge job with no DurableTaskClient present. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- .../GetLargePayloadTombstonesActivity.cs | 41 ++++- .../ReportLargePayloadPurgeResultsActivity.cs | 42 ++++- .../Client/GrpcLargePayloadPurgeClient.cs | 157 ------------------ .../Client/ILargePayloadPurgeClient.cs | 35 ---- ...rkerBuilderExtensions.AzureBlobPayloads.cs | 23 ++- .../UseExternalizedPayloadsTests.cs | 12 +- 6 files changed, 96 insertions(+), 214 deletions(-) delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Client/GrpcLargePayloadPurgeClient.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Client/ILargePayloadPurgeClient.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs index 3be72bc2..71322fed 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs @@ -1,8 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Grpc.Core; using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; +using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; +using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.AzureBlobPayloads; @@ -10,23 +13,49 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// Activity that fetches a bounded batch of due large-payload tombstones from the backend for the auto-purge /// job to delete. /// -/// The purge client used to query the backend for tombstones. +/// The sidecar service client used to query the backend for tombstones. /// The logger instance. [DurableTask] internal sealed class GetLargePayloadTombstonesActivity( - ILargePayloadPurgeClient client, + TaskHubSidecarServiceClient client, ILogger logger) : TaskActivity> { - readonly ILargePayloadPurgeClient client = Check.NotNull(client); + readonly TaskHubSidecarServiceClient client = Check.NotNull(client); readonly ILogger logger = Check.NotNull(logger); /// public override async Task> RunAsync(TaskActivityContext context, int input) { - int limit = input; - List tombstones = - await this.client.GetLargePayloadTombstonesAsync(limit, CancellationToken.None); + if (input <= 0 || input > 1000) + { + throw new ArgumentOutOfRangeException( + nameof(input), input, "Limit must be greater than 0 and less than or equal to 1000."); + } + + P.GetLargePayloadTombstonesResponse response; + try + { + response = await this.client.GetLargePayloadTombstonesAsync( + new P.GetLargePayloadTombstonesRequest { Limit = input }); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + "The GetLargePayloadTombstonesAsync operation was canceled.", e); + } + + List tombstones = new(response.Tombstones.Count); + foreach (P.LargePayloadTombstone tombstone in response.Tombstones) + { + tombstones.Add(new LargePayloadTombstone( + tombstone.PartitionId, + tombstone.InstanceKey, + tombstone.PayloadId, + tombstone.Token, + tombstone.Revision)); + } + this.logger.BlobPurgeFetchedTombstones(tombstones.Count); return tombstones; } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs index e65f88e9..87516635 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs @@ -1,8 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Grpc.Core; using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; +using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; +using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask.AzureBlobPayloads; @@ -12,15 +15,15 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// reported, not only the successful ones: the backend owns retry scheduling, so a row it hears nothing about /// would simply be re-served unchanged on the next cycle. /// -/// The purge client used to report purge results to the backend. +/// The sidecar service client used to report purge results to the backend. /// The logger instance. [DurableTask] internal sealed class ReportLargePayloadPurgeResultsActivity( - ILargePayloadPurgeClient client, + TaskHubSidecarServiceClient client, ILogger logger) : TaskActivity, object?> { - readonly ILargePayloadPurgeClient client = Check.NotNull(client); + readonly TaskHubSidecarServiceClient client = Check.NotNull(client); readonly ILogger logger = Check.NotNull(logger); /// @@ -32,7 +35,38 @@ internal sealed class ReportLargePayloadPurgeResultsActivity( return null; } - await this.client.ReportLargePayloadPurgeResultsAsync(input, CancellationToken.None); + P.ReportLargePayloadPurgeResultsRequest request = new(); + foreach (LargePayloadPurgeResult result in input) + { + request.Results.Add(new P.LargePayloadPurgeResult + { + PartitionId = result.PartitionId, + InstanceKey = result.InstanceKey, + PayloadId = result.PayloadId, + Revision = result.Revision, + + // The managed disposition enum declares the same numeric values as its protobuf counterpart, + // so it maps across by value. This is the only enum on the message and it only travels + // outbound, so the SDK can never receive a value it does not know. + Disposition = (P.LargePayloadPurgeDisposition)result.Disposition, + }); + } + + if (request.Results.Count == 0) + { + return null; + } + + try + { + await this.client.ReportLargePayloadPurgeResultsAsync(request); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + "The ReportLargePayloadPurgeResultsAsync operation was canceled.", e); + } + this.logger.BlobPurgeReportedResults(input.Count); return null; } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/GrpcLargePayloadPurgeClient.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/GrpcLargePayloadPurgeClient.cs deleted file mode 100644 index df5d9c08..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/GrpcLargePayloadPurgeClient.cs +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Grpc.Core; -using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Worker.Grpc; -using Microsoft.Extensions.Options; -using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; -using P = Microsoft.DurableTask.Protobuf; - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// gRPC implementation of that reuses the worker's own transport. It -/// builds a from the named , -/// so the two purge RPCs travel the exact same channel the worker already uses to reach the backend, with no -/// dependency on a client-side . The marshalling mirrors -/// GrpcDurableTaskClient's equivalent overrides so the wire behavior is identical either way. -/// -internal sealed class GrpcLargePayloadPurgeClient : ILargePayloadPurgeClient -{ - readonly IOptionsMonitor options; - readonly string name; - readonly object gate = new(); - TaskHubSidecarServiceClient? sidecarClient; - - /// - /// Initializes a new instance of the class. - /// - /// The monitor used to resolve the worker's gRPC options by builder name. - /// The builder name whose worker options carry the transport to reuse. - public GrpcLargePayloadPurgeClient(IOptionsMonitor options, string name) - { - this.options = Check.NotNull(options); - this.name = name ?? string.Empty; - } - - /// - public async Task> GetLargePayloadTombstonesAsync( - int limit, CancellationToken cancellation = default) - { - if (limit <= 0 || limit > 1000) - { - throw new ArgumentOutOfRangeException( - nameof(limit), limit, "Limit must be greater than 0 and less than or equal to 1000."); - } - - TaskHubSidecarServiceClient client = this.GetClient(); - - P.GetLargePayloadTombstonesResponse response; - try - { - response = await client.GetLargePayloadTombstonesAsync( - new P.GetLargePayloadTombstonesRequest { Limit = limit }, - cancellationToken: cancellation); - } - catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) - { - throw new OperationCanceledException( - $"The {nameof(this.GetLargePayloadTombstonesAsync)} operation was canceled.", e, cancellation); - } - - List result = new(response.Tombstones.Count); - foreach (P.LargePayloadTombstone tombstone in response.Tombstones) - { - result.Add(new LargePayloadTombstone( - tombstone.PartitionId, - tombstone.InstanceKey, - tombstone.PayloadId, - tombstone.Token, - tombstone.Revision)); - } - - return result; - } - - /// - public async Task ReportLargePayloadPurgeResultsAsync( - IEnumerable results, CancellationToken cancellation = default) - { - Check.NotNull(results); - - P.ReportLargePayloadPurgeResultsRequest request = new(); - foreach (LargePayloadPurgeResult result in results) - { - request.Results.Add(new P.LargePayloadPurgeResult - { - PartitionId = result.PartitionId, - InstanceKey = result.InstanceKey, - PayloadId = result.PayloadId, - Revision = result.Revision, - - // The managed disposition enum declares the same numeric values as its protobuf counterpart, - // so it maps across by value. This is the only enum on the message and it only travels - // outbound, so the SDK can never receive a value it does not know. - Disposition = (P.LargePayloadPurgeDisposition)result.Disposition, - }); - } - - if (request.Results.Count == 0) - { - return; - } - - TaskHubSidecarServiceClient client = this.GetClient(); - try - { - await client.ReportLargePayloadPurgeResultsAsync(request, cancellationToken: cancellation); - } - catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) - { - throw new OperationCanceledException( - $"The {nameof(this.ReportLargePayloadPurgeResultsAsync)} operation was canceled.", e, cancellation); - } - } - - TaskHubSidecarServiceClient GetClient() - { - // Resolve the transport lazily and cache it. The worker's Channel/CallInvoker is only guaranteed to be - // populated once the options have been fully post-configured (the AzureBlobPayloads worker setup moves - // the Channel onto CallInvoker, or throws if neither was set), which happens after this service is - // constructed. Building on first RPC keeps the purge activities constructible in hosts that never run - // the job - which is exactly the worker-only case this abstraction exists to unblock. - lock (this.gate) - { - if (this.sidecarClient is not null) - { - return this.sidecarClient; - } - - GrpcDurableTaskWorkerOptions opts = this.options.Get(this.name); - - // Mirror the worker's own Channel -> CallInvoker precedence (GrpcDurableTaskWorker.GetCallInvoker): - // reuse whichever the worker resolved so the purge RPCs share its exact transport. The worker also - // has an Address-only branch that builds an owned channel, but it is unreachable here - the - // AzureBlobPayloads worker PostConfigure guarantees one of these two is set or throws first - so we - // do not duplicate its per-target-framework channel-lifetime logic. - CallInvoker callInvoker; - if (opts.Channel is { } channel) - { - callInvoker = channel.CreateCallInvoker(); - } - else if (opts.CallInvoker is { } invoker) - { - callInvoker = invoker; - } - else - { - throw new InvalidOperationException( - "A gRPC Channel or CallInvoker must be configured on the worker to purge externalized payloads."); - } - - this.sidecarClient = new TaskHubSidecarServiceClient(callInvoker); - return this.sidecarClient; - } - } -} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/ILargePayloadPurgeClient.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/ILargePayloadPurgeClient.cs deleted file mode 100644 index 789ed911..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/ILargePayloadPurgeClient.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Microsoft.DurableTask.Client; - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// Abstraction over the two backend RPCs the auto-purge job needs: fetching a batch of due large-payload -/// tombstones and reporting the outcome of each attempted blob deletion. The worker talks to the backend over -/// its own gRPC transport through this abstraction, so the purge activities never depend on a -/// - which a worker-only ("client triggers, worker executes") host would never -/// register. -/// -internal interface ILargePayloadPurgeClient -{ - /// - /// Fetches a bounded batch of due large-payload tombstones from the backend. - /// - /// The maximum number of tombstones to return. Must be in the range 1..1000. - /// The cancellation token. - /// The due tombstones, up to of them. - Task> GetLargePayloadTombstonesAsync( - int limit, CancellationToken cancellation = default); - - /// - /// Reports the outcome of every attempted blob deletion to the backend so it can hard-delete the resolved - /// rows, reschedule the retryable ones, and quarantine the rest. - /// - /// The per-tombstone purge results to report. - /// The cancellation token. - /// A task that completes once the backend has accepted the report. - Task ReportLargePayloadPurgeResultsAsync( - IEnumerable results, CancellationToken cancellation = default); -} diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index 5606b864..38d73386 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Grpc.Core; using Grpc.Core.Interceptors; using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Worker; @@ -8,6 +9,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; +using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; using P = Microsoft.DurableTask.Protobuf; namespace Microsoft.DurableTask; @@ -102,20 +104,23 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB options.EnableEntitySupport = true; }); - // The purge activities fetch tombstones and report results over the worker's OWN gRPC transport, so a - // worker-only host (which never registers a DurableTaskClient) can still run the job. Resolve the - // transport from the same named GrpcDurableTaskWorkerOptions the worker uses, capturing builder.Name so a - // named worker binds to its own options. - builder.Services.TryAddSingleton(sp => + // The purge activities talk to the backend over the worker's OWN transport, so a worker-only host + // (which never registers a DurableTaskClient) can still run the job. This resolves the same CallInvoker + // the worker itself uses, so the RPCs ride its channel and interceptor - no second connection. + builder.Services.TryAddSingleton(sp => { - IOptionsMonitor options = - sp.GetRequiredService>(); - return new GrpcLargePayloadPurgeClient(options, builder.Name); + GrpcDurableTaskWorkerOptions options = + sp.GetRequiredService>().Get(builder.Name); + CallInvoker invoker = options.CallInvoker + ?? options.Channel?.CreateCallInvoker() + ?? throw new InvalidOperationException( + "A gRPC Channel or CallInvoker must be configured on the worker to purge externalized payloads."); + return new TaskHubSidecarServiceClient(invoker); }); // Register the entity/orchestrators/activities that run the singleton auto-purge job. These are // ALWAYS registered (not gated on AutoPurge) so that a client-enabled job always has something to - // execute here. The purge activities fetch/report via the injected ILargePayloadPurgeClient above. + // execute here. The purge activities fetch/report via the worker's TaskHubSidecarServiceClient above. builder.AddTasks(r => { r.AddEntity(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index 1ad2627a..a0c9eb11 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -205,11 +205,17 @@ public void UseExternalizedPayloads_WorkerOnly_PurgeActivitiesAreConstructible() // used to inject a concrete DurableTaskClient, which a worker-only host never registers, so they threw at // dispatch time - and only at dispatch, because DurableTaskRegistry stores a lazy // ActivatorUtilities.GetServiceOrCreateInstance factory - leaving auto-purge silently broken. They now - // inject the worker's own ILargePayloadPurgeClient, so they construct with no DurableTaskClient present. + // inject the worker's own TaskHubSidecarServiceClient (built from the worker's GrpcDurableTaskWorkerOptions), + // so they construct with no DurableTaskClient present. A real worker configures a transport, so UseGrpc + // supplies one here - without it the worker's PostConfigure would throw for having neither Channel nor + // CallInvoker. ServiceCollection services = new(); services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); - services.AddDurableTaskWorker( - builder => builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true")); + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(options => options.CallInvoker = Mock.Of()); + builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + }); using ServiceProvider provider = services.BuildServiceProvider(); From 00cbb6861c45325600a27178cc785cf44fae7887 Mon Sep 17 00:00:00 2001 From: wangbill Date: Sun, 16 Aug 2026 16:02:39 -0400 Subject: [PATCH 37/47] Disable blob auto-purge on backends missing the purge RPCs During a mixed rollout an older DTS backend (or a stale local emulator image) returns gRPC Unimplemented for the new large-payload purge RPCs. The fetch/report activities previously translated only Cancelled, so the purge job retried the unsupported operation forever (~1m45s loop) with no actionable diagnostic. Map Unimplemented to NotImplementedException and genuinely stop the job via a new terminal Unsupported state. - BlobPurgeJobStatus: add terminal Unsupported, distinct from Pending so Create refuses to re-activate it. - BlobPurgeJob: Create guards against Unsupported (the reconciler re-issues Create every interval, so this guard is what makes the stop real); add idempotent MarkUnsupported and guarded Reset ops. - Both purge activities and both GrpcDurableTaskClient overrides map StatusCode.Unimplemented -> NotImplementedException (client overrides follow the existing bare-Detail precedent). - Orchestrator: retry policy HandleFailure skips NotImplementedException; a dedicated catch disables the job via awaited MarkUnsupported and exits the loop cleanly with a dedicated diagnostic. - BlobPurgeJobStarter: signal Reset once per process start so an upgraded backend recovers on restart (benign, self-correcting race documented). - Logs: EventIds 827-830. Tests: entity Create-does-not-resurrect (failing-first), MarkUnsupported idempotency, Reset clears/no-ops on Active; activity + client Unimplemented mapping; orchestrator disables-and-exits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- src/Client/Grpc/GrpcDurableTaskClient.cs | 8 + .../GetLargePayloadTombstonesActivity.cs | 12 ++ .../ReportLargePayloadPurgeResultsActivity.cs | 12 ++ .../AutoPurge/Client/BlobPurgeJobStarter.cs | 32 ++++ .../AutoPurge/Entity/BlobPurgeJob.cs | 68 ++++++++ .../AzureBlobPayloads/AutoPurge/Logs.cs | 12 ++ .../AutoPurge/Models/BlobPurgeJobStatus.cs | 10 ++ .../BlobPurgeJobOrchestrator.cs | 21 ++- .../LargePayloadPurgeUnimplementedTests.cs | 92 +++++++++++ .../BlobPurgeJobOrchestratorTests.cs | 58 +++++++ .../AutoPurge/BlobPurgeJobTests.cs | 156 ++++++++++++++++++ .../PurgeActivityUnimplementedTests.cs | 89 ++++++++++ 12 files changed, 569 insertions(+), 1 deletion(-) create mode 100644 test/Client/Grpc.Tests/LargePayloadPurgeUnimplementedTests.cs create mode 100644 test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeActivityUnimplementedTests.cs diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 0fdbf161..9a5fcb98 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -641,6 +641,10 @@ public override async Task> GetLargePayloadTombstone new P.GetLargePayloadTombstonesRequest { Limit = limit }, cancellationToken: cancellation); } + catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) + { + throw new NotImplementedException(e.Status.Detail); + } catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) { throw new OperationCanceledException( @@ -693,6 +697,10 @@ public override async Task ReportLargePayloadPurgeResultsAsync( { await this.sidecarClient.ReportLargePayloadPurgeResultsAsync(request, cancellationToken: cancellation); } + catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) + { + throw new NotImplementedException(e.Status.Detail); + } catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) { throw new OperationCanceledException( diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs index 71322fed..0d9dc78c 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs @@ -44,6 +44,18 @@ public override async Task> RunAsync(TaskActivityCon throw new OperationCanceledException( "The GetLargePayloadTombstonesAsync operation was canceled.", e); } + catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) + { + // Mixed-rollout guard: an older backend build (or a stale local emulator image) does not implement + // this RPC. Surfacing NotImplementedException lets the orchestrator disable the job instead of + // retrying an operation that can never succeed - see BlobPurgeJobOrchestrator's handling of it. + throw new NotImplementedException( + "The Durable Task backend does not implement the GetLargePayloadTombstones RPC required for " + + "large-payload auto-purge. Upgrade the backend (or re-pull " + + "'mcr.microsoft.com/dts/dts-emulator'), then restart the app. Auto-purge is now disabled until " + + $"the process restarts. Backend detail: {e.Status.Detail}", + e); + } List tombstones = new(response.Tombstones.Count); foreach (P.LargePayloadTombstone tombstone in response.Tombstones) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs index 87516635..a1409a01 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs @@ -66,6 +66,18 @@ internal sealed class ReportLargePayloadPurgeResultsActivity( throw new OperationCanceledException( "The ReportLargePayloadPurgeResultsAsync operation was canceled.", e); } + catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) + { + // Mixed-rollout guard: an older backend build (or a stale local emulator image) does not implement + // this RPC. Surfacing NotImplementedException lets the orchestrator disable the job instead of + // retrying an operation that can never succeed - see BlobPurgeJobOrchestrator's handling of it. + throw new NotImplementedException( + "The Durable Task backend does not implement the ReportLargePayloadPurgeResults RPC required " + + "for large-payload auto-purge. Upgrade the backend (or re-pull " + + "'mcr.microsoft.com/dts/dts-emulator'), then restart the app. Auto-purge is now disabled until " + + $"the process restarts. Backend detail: {e.Status.Detail}", + e); + } this.logger.BlobPurgeReportedResults(input.Count); return null; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 97807479..a894491c 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -180,6 +180,38 @@ public void Dispose() async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationToken cancellationToken) { + // Clear a stale "unsupported" disable exactly once per process start, before the reconcile loop begins. + // If an earlier run of this process disabled the job because the backend lacked the purge RPCs, the + // entity is still Unsupported and Create keeps no-opping (that guard is what makes the disable real). + // Reset moves it back to Pending so the first reconcile pass can re-check a possibly-upgraded backend, + // giving a cheap, discoverable recovery: upgrade the backend / re-pull the emulator, then restart. + // + // Reset is guarded in the entity to touch only an Unsupported job, so this blind startup signal is a + // no-op against a healthy Active job - it can never stop one. Signalled rather than driven through the + // bridge orchestration, matching SignalJobStopAsync: unlike Create it needs no fixed-instance dedupe and + // is idempotent in the entity. + // + // Known, benign race: this signal and the first pass's Create below are two independent round trips with + // nothing ordering them. If Create lands first it sees the entity still Unsupported and no-ops, and the + // job comes back on the NEXT reconcile pass (within one ReconcileInterval). That is harmless and + // self-correcting; this comment exists so a future reader does not "fix" it by adding coordination. + try + { + await client.Entities.SignalEntityAsync( + this.entityId, nameof(BlobPurgeJob.Reset), cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + // Never let a failed reset prevent the reconcile loop from starting. A lost reset only delays + // recovery from a prior unsupported disable by one process lifetime; not starting the loop would + // mean the job is never (re)created at all. Log and fall through. + this.logger.BlobPurgeStarterRetry(ex); + } + while (!cancellationToken.IsCancellationRequested) { TimeSpan delay; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index f1b704e7..b3f06aac 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -24,6 +24,17 @@ class BlobPurgeJob(ILogger logger) : TaskEntity /// public void Create(TaskEntityContext context, int purgeBatchSize) { + if (this.State.Status == BlobPurgeJobStatus.Unsupported) + { + // This guard is the ONLY thing that makes the "unsupported" stop real. BlobPurgeJobStarter re-issues + // Create on every reconciliation pass (roughly every ReconcileInterval), so any status Create is + // willing to revive would have the job back within minutes and it would spin against the missing RPC + // forever - the disable would be theater. Recovery is deliberately by process restart, which signals + // Reset before the first Create; do NOT signal Run here. + logger.BlobPurgeJobCreateSkippedUnsupported(context.Id.Key); + return; + } + if (this.State.Status == BlobPurgeJobStatus.Active) { // The batch size is taken because this is the only path by which a changed configuration reaches an @@ -156,6 +167,63 @@ public void Stop(TaskEntityContext context) logger.BlobPurgeJobStopped(context.Id.Key); } + /// + /// Marks the job because the backend does not implement the + /// large-payload purge RPCs. + /// + /// + /// This is a genuine terminal stop, not a pause. The orchestrator reaches it after the fetch or report + /// activity surfaces a gRPC Unimplemented as a , and unlike + /// the resulting status is one that refuses to re-activate. That is + /// the whole point: the client-side starter re-issues Create every reconcile interval, so any status + /// Create is willing to revive would have the job back within minutes. Recovery is by process + /// restart, which re-checks the backend and clears this via . + /// + /// The entity context. + /// A human-readable description of why the backend is unsupported. + public void MarkUnsupported(TaskEntityContext context, string detail) + { + if (this.State.Status == BlobPurgeJobStatus.Unsupported) + { + // Idempotent no-op. The orchestrator awaits this call and then exits, but concurrent orchestrators + // (one per replica, all hitting the same unsupported backend) can each report before the others + // exit. Leaving the state untouched on the repeat keeps LastModifiedAt meaning "when the job was + // disabled" rather than "when the last replica noticed", mirroring the guard discipline on Stop. + return; + } + + this.State.Status = BlobPurgeJobStatus.Unsupported; + this.State.LastError = detail; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.BlobPurgeJobMarkedUnsupported(context.Id.Key, detail); + } + + /// + /// Clears an disable so the job can be created again after the + /// process restarts against a backend that may now implement the purge RPCs. + /// + /// + /// The guard is load-bearing. The client-side starter signals this blind exactly once per process start, + /// before it knows the job's state, so it must be incapable of disturbing anything other than an unsupported + /// job. In particular a signal landing on a healthy job must be a + /// no-op: without the guard, a routine host restart would stop a running job every time. + /// + /// The entity context. + public void Reset(TaskEntityContext context) + { + if (this.State.Status != BlobPurgeJobStatus.Unsupported) + { + return; + } + + this.State.Status = BlobPurgeJobStatus.Pending; + this.State.LastError = null; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.BlobPurgeJobReset(context.Id.Key); + } + /// /// Records progress after a purge cycle completes. /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 3f3ae217..66aadd15 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -60,4 +60,16 @@ static partial class Logs [LoggerMessage(EventId = 826, Level = LogLevel.Warning, Message = "Blob payload auto-purge is disabled but the singleton job's state could not be read; requesting a stop anyway. Stopping a job that may be deleting payloads matters more than avoiding a redundant stop request, which the job ignores.")] public static partial void BlobPurgeJobStateUnknown(this ILogger logger, Exception exception); + + [LoggerMessage(EventId = 827, Level = LogLevel.Warning, Message = "Blob payload auto-purge job '{jobId}' was disabled because the backend does not implement the large-payload purge RPCs: {detail}. This is expected against an older backend build or a stale local emulator image. Upgrade the Durable Task backend (or re-pull 'mcr.microsoft.com/dts/dts-emulator'), then restart the app; the job stays disabled until the process restarts.")] + public static partial void BlobPurgeJobMarkedUnsupported(this ILogger logger, string? jobId, string detail); + + [LoggerMessage(EventId = 828, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' was reset from the unsupported state on process start, so it will be created again and re-check whether the backend now implements the purge RPCs.")] + public static partial void BlobPurgeJobReset(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 829, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is marked unsupported, so the create request was ignored. The job stays disabled until the process restarts and clears the unsupported state.")] + public static partial void BlobPurgeJobCreateSkippedUnsupported(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 830, Level = LogLevel.Error, Message = "Blob payload auto-purge for job '{jobId}' stopped because the backend does not implement the large-payload purge RPCs: {detail}. The job is now disabled and will not delete blobs. Upgrade the Durable Task backend (or re-pull 'mcr.microsoft.com/dts/dts-emulator'), then restart the app to resume auto-purge.")] + public static partial void BlobPurgeBackendUnsupported(this ILogger logger, string? jobId, string detail); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs index fe6fe99c..8d1eefc2 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs @@ -20,4 +20,14 @@ public enum BlobPurgeJobStatus /// The job is active and draining tombstoned payloads from the backend. /// Active, + + /// + /// The backend does not implement the large-payload purge RPCs, so the job cannot run. This happens against + /// an older backend build or a stale local emulator image that predates the feature. It is kept distinct + /// from precisely because must NOT re-activate it: + /// the client-side starter re-issues Create on every reconciliation pass, so a merely-stopped + /// () job is meant to come back, whereas an unsupported one must stay down until the + /// process restarts and re-checks the backend. + /// + Unsupported = 2, } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index a4e455e4..100bfc66 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -35,7 +35,13 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator !details.IsCausedBy(), + }; /// public override async Task RunAsync(TaskOrchestrationContext context, BlobPurgeJobRunRequest input) @@ -128,6 +134,19 @@ await context.Entities.CallEntityAsync( await context.CreateTimer(ErrorBackoff, default); } } + catch (TaskFailedException ex) when (ex.FailureDetails.IsCausedBy()) + { + // The backend does not implement the large-payload purge RPCs (an older backend build or a + // stale local emulator image). Retrying cannot help, so disable the job durably and exit the + // perpetual loop cleanly instead of logging a generic failure and backing off forever. + // MarkUnsupported is AWAITED, not signalled, so the disable is committed to the entity before we + // return; the client-side starter will not resurrect an Unsupported job until the process + // restarts. This is deliberately a distinct diagnostic from BlobPurgeCycleFailed below. + logger.BlobPurgeBackendUnsupported(jobId, ex.FailureDetails.ErrorMessage); + await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.MarkUnsupported), ex.FailureDetails.ErrorMessage); + return null; + } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // A single bad cycle (transient backend/entity/activity failure) must not kill the perpetual diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeUnimplementedTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeUnimplementedTests.cs new file mode 100644 index 00000000..dc36da7b --- /dev/null +++ b/test/Client/Grpc.Tests/LargePayloadPurgeUnimplementedTests.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.Client.Grpc.Tests; + +/// +/// Public-API parity for the mixed-rollout mapping: the two large-payload purge overrides on +/// must translate a gRPC Unimplemented from an older backend into a +/// , matching the precedent already used elsewhere in the client for +/// unsupported RPCs. +/// +public class LargePayloadPurgeUnimplementedTests +{ + readonly Mock loggerMock = new(); + + [Fact] + public async Task GetLargePayloadTombstonesAsync_WhenBackendUnimplemented_ThrowsNotImplemented() + { + // Arrange - the backend does not implement the fetch RPC. + GrpcDurableTaskClient client = this.CreateThrowingClient( + new RpcException(new Status(StatusCode.Unimplemented, "unknown method"))); + + // Act + Func act = () => client.GetLargePayloadTombstonesAsync(100); + + // Assert + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task ReportLargePayloadPurgeResultsAsync_WhenBackendUnimplemented_ThrowsNotImplemented() + { + // Arrange - a non-empty result list is required to reach the RPC past the empty-request short-circuit. + GrpcDurableTaskClient client = this.CreateThrowingClient( + new RpcException(new Status(StatusCode.Unimplemented, "unknown method"))); + LargePayloadPurgeResult[] results = + { + new(1, 2, 3, 4, LargePayloadPurgeDisposition.Deleted), + }; + + // Act + Func act = () => client.ReportLargePayloadPurgeResultsAsync(results); + + // Assert + await act.Should().ThrowAsync(); + } + + GrpcDurableTaskClient CreateThrowingClient(RpcException error) + { + GrpcDurableTaskClientOptions options = new() + { + CallInvoker = new ThrowingCallInvoker(error), + }; + + return new GrpcDurableTaskClient("test", options, this.loggerMock.Object); + } + + /// + /// A minimal whose unary calls fault with a configured , + /// so the client's real RPC path runs and hits its own catch clause exactly as a live channel would. + /// + sealed class ThrowingCallInvoker(RpcException error) : CallInvoker + { + public override AsyncUnaryCall AsyncUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => new( + Task.FromException(error), + Task.FromResult(new Metadata()), + () => error.Status, + () => new Metadata(), + () => { }); + + public override TResponse BlockingUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => throw error; + + public override AsyncServerStreamingCall AsyncServerStreamingCall( + Method method, string? host, CallOptions options, TRequest request) + => throw new NotSupportedException(); + + public override AsyncClientStreamingCall AsyncClientStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + + public override AsyncDuplexStreamingCall AsyncDuplexStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs index 609cdcb9..db1b72aa 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs @@ -71,6 +71,64 @@ public async Task RunAsync_WhenJobIsStopped_ExitsWithoutFetching() this.logger.Logs.Should().Contain(entry => entry.Message.Contains("stopping")); } + [Fact] + public async Task RunAsync_WhenBackendDoesNotImplementPurgeRpcs_DisablesJobAndExits() + { + // Arrange - the fetch activity surfaced a gRPC Unimplemented as NotImplementedException (mixed rollout / + // stale emulator). The orchestrator must disable the job durably and exit its perpetual loop, rather + // than logging a generic cycle failure and retrying on every backoff forever. + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 250 }); + + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + + context + .Setup(c => c.CallActivityAsync>( + It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new TaskFailedException( + nameof(GetLargePayloadTombstonesActivity), + 1, + new NotImplementedException("backend does not implement GetLargePayloadTombstones"))); + + // Act + object? result = await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert - disabled through an AWAITED MarkUnsupported so the write is durable before the loop exits, a + // dedicated diagnostic is logged (not the generic cycle-failed one), and RunAsync returns rather than + // continuing. The awaited call, not a signal, is what guarantees the disable is committed here. + result.Should().BeNull(); + entities.Verify( + e => e.CallEntityAsync( + JobEntityId, + nameof(BlobPurgeJob.MarkUnsupported), + It.IsAny(), + It.IsAny()), + Times.Once); + this.logger.Logs.Should().Contain( + entry => entry.Message.Contains("does not implement the large-payload purge RPCs")); + this.AssertNoCycleFailed(); + } + /// /// Guards against the whole test passing through the orchestrator's catch-all cycle handler, which would /// leave every observation empty and make the assertions vacuous. diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 88d4fa6a..9996687d 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -367,4 +367,160 @@ public async Task Get_ReturnsCurrentState() state.Status.Should().Be(BlobPurgeJobStatus.Active); state.PurgeBatchSize.Should().Be(42); } + + [Fact] + public async Task Create_WhenUnsupported_DoesNotReactivate() + { + // Arrange - the backend was found not to implement the purge RPCs, so the job is disabled. Create runs + // on every reconciliation pass, so if it revived an unsupported job the disable would be undone within + // one interval and the job would spin against the missing RPC forever. This guard is the only thing that + // makes the stop real; recovery is deliberately by process restart (via Reset), not by Create. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Unsupported, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 250); + + // Act + await this.job.RunAsync(operation); + + // Assert - the status is the load-bearing assertion: without the guard Create falls through and sets it + // Active. Run must not be signalled either, since that is the other half of a revival. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Unsupported); + + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task MarkUnsupported_WhenActive_DisablesJobAndRecordsDetail() + { + // Arrange - a running job whose fetch/report activity just surfaced a gRPC Unimplemented. The job must + // move to a status Create refuses to revive, and the detail is retained so an operator can see why. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.MarkUnsupported), + new TestEntityState(existing), + "backend does not implement GetLargePayloadTombstones"); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Unsupported); + state.LastError.Should().Be("backend does not implement GetLargePayloadTombstones"); + state.LastModifiedAt.Should().NotBeNull(); + + // Unlike Create, this must not (re)start the orchestrator - the job is being disabled, not run. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task MarkUnsupported_WhenAlreadyUnsupported_LeavesStateUntouched() + { + // Arrange - a second replica reporting the same unsupported backend before the first orchestrator has + // exited. The repeat must be a no-op so LastModifiedAt keeps meaning "when the job was disabled" and the + // original detail is not overwritten by a later, possibly less specific, one. + DateTimeOffset disabledAt = DateTimeOffset.UtcNow.AddMinutes(-5); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Unsupported, + LastError = "original detail", + LastModifiedAt = disabledAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.MarkUnsupported), + new TestEntityState(existing), + "a different detail"); + + // Act + await this.job.RunAsync(operation); + + // Assert - the sibling test above is the positive control: identical wiring off an Active job does move + // both fields, proving this no-op is the guard's doing and not a dead path. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Unsupported); + state.LastError.Should().Be("original detail"); + state.LastModifiedAt.Should().Be(disabledAt); + } + + [Fact] + public async Task Reset_WhenUnsupported_MovesToPendingAndClearsError() + { + // Arrange - a process starting up against a backend that may have been upgraded. Reset clears the + // unsupported disable so the next Create can re-check the backend, and drops the stale error. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Unsupported, + LastError = "backend does not implement GetLargePayloadTombstones", + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Reset), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Pending); + state.LastError.Should().BeNull(); + state.LastModifiedAt.Should().NotBeNull(); + } + + [Fact] + public async Task Reset_WhenActive_LeavesStateUntouched() + { + // Arrange - the worst regression this guard prevents. Reset is signalled blind on every process start, + // before the starter knows the job's state, so a signal landing on a healthy Active job must change + // nothing. Without the guard, a routine host restart would stop a running job every time. + DateTimeOffset activeSince = DateTimeOffset.UtcNow.AddHours(-3); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = activeSince, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Reset), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.LastModifiedAt.Should().Be(activeSince); + } } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeActivityUnimplementedTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeActivityUnimplementedTests.cs new file mode 100644 index 00000000..82a02e33 --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/PurgeActivityUnimplementedTests.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Grpc.Core; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using static Microsoft.DurableTask.Protobuf.TaskHubSidecarService; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +/// +/// The two purge activities are the real runtime path to the backend - they call the generated sidecar client +/// directly. During a mixed rollout an older backend returns gRPC Unimplemented for the new purge RPCs; +/// each activity must translate that into a so the orchestrator can +/// disable the job instead of retrying an operation that can never succeed. +/// +public class PurgeActivityUnimplementedTests +{ + [Fact] + public async Task GetLargePayloadTombstones_WhenBackendUnimplemented_ThrowsNotImplemented() + { + // Arrange - the backend rejects the fetch RPC because it does not implement it. + TaskHubSidecarServiceClient client = new( + new ThrowingCallInvoker(new RpcException(new Status(StatusCode.Unimplemented, "unknown method")))); + GetLargePayloadTombstonesActivity activity = new(client, new TestLogger()); + + // Act + Func act = () => activity.RunAsync(null!, 100); + + // Assert - a bare RpcException would let the retry policy spin forever; NotImplementedException is the + // signal the orchestrator matches on to stop. + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task ReportLargePayloadPurgeResults_WhenBackendUnimplemented_ThrowsNotImplemented() + { + // Arrange - a non-empty result list is required to get past the activity's empty-input short-circuit and + // actually reach the report RPC, which the backend does not implement. + TaskHubSidecarServiceClient client = new( + new ThrowingCallInvoker(new RpcException(new Status(StatusCode.Unimplemented, "unknown method")))); + ReportLargePayloadPurgeResultsActivity activity = + new(client, new TestLogger()); + List results = new() + { + new LargePayloadPurgeResult(1, 2, 3, 4, LargePayloadPurgeDisposition.Deleted), + }; + + // Act + Func act = () => activity.RunAsync(null!, results); + + // Assert + await act.Should().ThrowAsync(); + } + + /// + /// A minimal whose unary calls fault with a configured . + /// Building a real over it exercises the activity's own catch + /// clause exactly as a live channel would, without mocking the generated client. The faulted + /// surfaces the exception on await, matching real gRPC. + /// + sealed class ThrowingCallInvoker(RpcException error) : CallInvoker + { + public override AsyncUnaryCall AsyncUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => new( + Task.FromException(error), + Task.FromResult(new Metadata()), + () => error.Status, + () => new Metadata(), + () => { }); + + public override TResponse BlockingUnaryCall( + Method method, string? host, CallOptions options, TRequest request) + => throw error; + + public override AsyncServerStreamingCall AsyncServerStreamingCall( + Method method, string? host, CallOptions options, TRequest request) + => throw new NotSupportedException(); + + public override AsyncClientStreamingCall AsyncClientStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + + public override AsyncDuplexStreamingCall AsyncDuplexStreamingCall( + Method method, string? host, CallOptions options) + => throw new NotSupportedException(); + } +} From d146c506b249c2d2c87c0cce0f7f346a1c1acea0 Mon Sep 17 00:00:00 2001 From: wangbill Date: Sun, 16 Aug 2026 16:41:23 -0400 Subject: [PATCH 38/47] Batch blob-purge deletes into chunks and bound each delete's wall-clock time Addresses two performance findings in the large-payload blob auto-purge job. Thread 1 (per-blob activity fan-out): the orchestrator previously issued one CallActivityAsync per tombstone. A 1000-row batch meant 1000 activity calls (~2000 history events) per cycle. DeleteExternalBlobActivity now accepts a chunk of tokens (List) and returns one outcome per token (List), and the orchestrator fans out in chunks of DeleteChunkSize=50 -> 20 activity calls (~40 history events), a 50x reduction. Total concurrent storage deletes are held at today's 32: MaxParallelChunkActivities=4 (orchestrator) x MaxParallelDeletesPerChunk=8 (inside each chunk activity) = 32. Results stay per-token: the chunk activity writes each outcome into a pre-sized array by input index, and the orchestrator asserts the returned count equals the sent count before zipping so a mismatch fails loudly instead of misattributing dispositions. One failing token in a chunk never fails its peers (every branch still returns a disposition; the activity never throws). Chunk-level retry stays safe because deletion is idempotent. The per-token classification body is moved verbatim; this is a fan-out change, not a semantics change. The Azure Blob Batch suggestion is intentionally not implemented (extra package + bulk store API + per-sub-request status semantics; the chunking already removes the orchestration-history problem). Thread 2 (unbounded per-delete retry budget): a single hung blob delete could burn up to ~18 minutes (9 attempts x 2 min) while holding a parallelism slot, because the store delete was called with CancellationToken.None. Each delete is now bounded by a CancellationTokenSource with SingleDeleteTimeout=60s. A timeout surfaces as TaskCanceledException, which the existing catch-all maps to a Retry disposition, so a persistently slow blob backs off server-side instead of stalling the wave. TaskActivityContext exposes no ambient CancellationToken, so the timeout source stands alone (host shutdown still tears the activity down). PUBLIC API BREAK (unreleased feature): DeleteExternalBlobActivity changes from TaskActivity to TaskActivity, List>. No public-API baseline file exists in the repo, so nothing to update. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- .../Activities/DeleteExternalBlobActivity.cs | 72 ++++++++- .../BlobPurgeJobOrchestrator.cs | 90 ++++++++--- .../BlobPurgeJobOrchestratorTests.cs | 97 ++++++++++++ .../DeleteExternalBlobActivityTests.cs | 145 ++++++++++++++++-- 4 files changed, 358 insertions(+), 46 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 4d74b601..7085e1ad 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -10,10 +10,11 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Activity that deletes a single externalized payload blob given its token, and classifies the attempt as +/// Activity that deletes a chunk of externalized payload blobs given their tokens, classifying each attempt as /// , , or -/// . Deletion is idempotent, so re-delivered tokens and -/// concurrent workers are safe. +/// and returning one outcome per token, positionally +/// aligned to the input. Deleting a whole chunk in a single activity call is what keeps orchestration history +/// small. Deletion is idempotent, so re-delivered tokens, a retried chunk, and concurrent workers are all safe. /// /// /// The split between retry and quarantine is whether the failure can self-heal, verified against the @@ -50,17 +51,66 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class DeleteExternalBlobActivity( PayloadStore store, ILogger logger) - : TaskActivity + : TaskActivity, List> { + // Concurrency cap for the deletes WITHIN one chunk. The orchestrator runs at most + // BlobPurgeJobOrchestrator.MaxParallelChunkActivities (4) of these activities at once, so the total number + // of storage deletes in flight across the worker is 4 x 8 = 32 - identical to the flat cap this chunked + // design replaced. That product is the real budget: if either factor changes the other must move to keep it + // at 32, or the worker will either starve throughput or multiply into hundreds of concurrent storage calls + // (e.g. 20 chunks x 32 = 640). + const int MaxParallelDeletesPerChunk = 8; + readonly PayloadStore store = Check.NotNull(store); readonly ILogger logger = Check.NotNull(logger); + /// + /// Gets or sets the wall-clock ceiling for a single blob delete. The store's own retry policy allows up to + /// 8 attempts against a 2-minute network timeout (~18 minutes worst case) for one blob, and a chunk awaits + /// its slowest delete, so without a bound one hung blob would hold a concurrency slot for many minutes and + /// stall the whole wave. Capping it well under the activity's own 15s/30s/60s retry cadence means a stuck + /// delete gives up, surfaces as , and is classified + /// for the backend to defer - rather than pinning the slot. + /// It is settable only so a test can shrink it; it is never reconfigured at runtime. + /// + internal TimeSpan SingleDeleteTimeout { get; set; } = TimeSpan.FromSeconds(60); + /// - public override async Task RunAsync(TaskActivityContext context, string input) + public override async Task> RunAsync(TaskActivityContext context, List input) { - Check.NotNullOrEmpty(input, nameof(input)); + Check.NotNull(input, nameof(input)); + + // Write each outcome at its token's INDEX, never in completion order: the deletes below run + // concurrently and the orchestrator zips these back onto tombstones positionally, so a delete finishing + // out of order must not shift a disposition onto the wrong row. + BlobPurgeOutcome[] outcomes = new BlobPurgeOutcome[input.Count]; + + // A single bad token never fails its peers: DeleteAsync returns a disposition on every branch instead + // of throwing (its catch-all absorbs everything but OutOfMemory/StackOverflow), so Task.WhenAll never + // observes a fault from a classified failure and the other tokens in the chunk still complete and report. + using SemaphoreSlim gate = new(MaxParallelDeletesPerChunk, MaxParallelDeletesPerChunk); + Task[] deletes = new Task[input.Count]; + for (int i = 0; i < input.Count; i++) + { + deletes[i] = DeleteAtAsync(i); + } - return await this.DeleteAsync(input); + await Task.WhenAll(deletes); + + return new List(outcomes); + + async Task DeleteAtAsync(int index) + { + await gate.WaitAsync().ConfigureAwait(false); + try + { + outcomes[index] = await this.DeleteAsync(input[index]).ConfigureAwait(false); + } + finally + { + gate.Release(); + } + } } /// @@ -109,7 +159,13 @@ async Task DeleteAsync(string token) try { - PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, CancellationToken.None); + // Bound the delete's wall-clock time (see SingleDeleteTimeout). If it elapses, the token cancels the + // store call, which surfaces as OperationCanceledException and is classified retryable by the + // catch-all below - so a hung blob yields to the backend's deferral instead of pinning its slot. + // TaskActivityContext exposes no ambient cancellation token to link, so this timeout is the only + // cancellation source; host shutdown is handled by the worker tearing the activity down. + using CancellationTokenSource timeout = new(this.SingleDeleteTimeout); + PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, timeout.Token); // The blob exists but this store never wrote it, so it was left untouched. That is an expected // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it would diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 100bfc66..30e29de9 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -26,7 +26,18 @@ public sealed record BlobPurgeJobRunRequest( public class BlobPurgeJobOrchestrator : TaskOrchestrator { const int ContinueAsNewFrequency = 5; - const int MaxParallelDeletes = 32; + + // Tombstones are deleted a chunk at a time: one activity call handles DeleteChunkSize tokens instead of a + // single token, which is what keeps orchestration history small. A full 1000-row cycle becomes 1000 / 50 = + // 20 activity calls (~40 history events) instead of 1000 calls (~2000 events) - a 50x reduction - and drains + // in ceil(20 / 4) = 5 waves instead of 32. + const int DeleteChunkSize = 50; + + // How many chunk-delete activities run concurrently. TOTAL concurrent storage deletes are this times the + // activity's own MaxParallelDeletesPerChunk (8): 4 x 8 = 32, exactly the flat cap the per-token design used. + // The product is the budget - moving one factor without the other either starves throughput or multiplies + // into hundreds of concurrent storage calls. + const int MaxParallelChunkActivities = 4; static readonly TimeSpan IdleDelay = TimeSpan.FromMinutes(1); static readonly TimeSpan ErrorBackoff = TimeSpan.FromMinutes(1); @@ -174,53 +185,82 @@ static int CountDisposition( } static async Task DrainAsync( - List> tasks, List results) + List>> tasks, List results) { - LargePayloadPurgeResult[] completed = await Task.WhenAll(tasks); - results.AddRange(completed); + List[] completed = await Task.WhenAll(tasks); + foreach (List chunkResults in completed) + { + results.AddRange(chunkResults); + } } async Task> DeleteBatchAsync( TaskOrchestrationContext context, List tombstones) { List results = new(tombstones.Count); - List> tasks = new(); + List>> inFlight = new(); - foreach (LargePayloadTombstone tombstone in tombstones) + for (int start = 0; start < tombstones.Count; start += DeleteChunkSize) { - tasks.Add(this.DeleteOneAsync(context, tombstone)); + int count = Math.Min(DeleteChunkSize, tombstones.Count - start); + List chunk = tombstones.GetRange(start, count); + inFlight.Add(this.DeleteChunkAsync(context, chunk)); - if (tasks.Count >= MaxParallelDeletes) + if (inFlight.Count >= MaxParallelChunkActivities) { - await DrainAsync(tasks, results); - tasks.Clear(); + await DrainAsync(inFlight, results); + inFlight.Clear(); } } - if (tasks.Count > 0) + if (inFlight.Count > 0) { - await DrainAsync(tasks, results); + await DrainAsync(inFlight, results); } return results; } - async Task DeleteOneAsync( - TaskOrchestrationContext context, LargePayloadTombstone tombstone) + async Task> DeleteChunkAsync( + TaskOrchestrationContext context, List chunk) { - BlobPurgeOutcome outcome = await context.CallActivityAsync( + List tokens = new(chunk.Count); + foreach (LargePayloadTombstone tombstone in chunk) + { + tokens.Add(tombstone.Token); + } + + List outcomes = await context.CallActivityAsync>( nameof(DeleteExternalBlobActivity), - tombstone.Token, + tokens, new TaskOptions(PurgeActivityRetryPolicy)); - // The revision is echoed back unchanged so the backend can detect a tombstone that was rewritten while - // this attempt was in flight and ignore the stale result. Retry scheduling is the backend's job, so no - // next-attempt time is computed here. - return new LargePayloadPurgeResult( - tombstone.PartitionId, - tombstone.InstanceKey, - tombstone.PayloadId, - tombstone.Revision, - outcome.Disposition); + // The activity contract is one outcome per input token, positionally aligned. Assert that before zipping + // so a broken contract fails loudly here instead of silently pinning each disposition onto the wrong row + // - which would tell the backend to delete, retry, or quarantine the wrong payloads. + if (outcomes is null || outcomes.Count != chunk.Count) + { + throw new InvalidOperationException( + $"The blob delete activity returned {outcomes?.Count ?? 0} outcomes for a chunk of {chunk.Count} " + + "tokens; expected exactly one per token. Refusing to attribute dispositions to the wrong rows."); + } + + List results = new(chunk.Count); + for (int i = 0; i < chunk.Count; i++) + { + LargePayloadTombstone tombstone = chunk[i]; + + // The revision is echoed back unchanged so the backend can detect a tombstone that was rewritten + // while this attempt was in flight and ignore the stale result. Retry scheduling is the backend's + // job, so no next-attempt time is computed here. + results.Add(new LargePayloadPurgeResult( + tombstone.PartitionId, + tombstone.InstanceKey, + tombstone.PayloadId, + tombstone.Revision, + outcomes[i].Disposition)); + } + + return results; } } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs index db1b72aa..9f392ab2 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs @@ -129,6 +129,103 @@ public async Task RunAsync_WhenBackendDoesNotImplementPurgeRpcs_DisablesJobAndEx this.AssertNoCycleFailed(); } + [Fact] + public async Task RunAsync_DeletesLargeBatch_InChunksNotOneActivityPerToken() + { + // Arrange - a full 1000-row batch. The whole point of chunking is that a batch this size fans out to a + // HANDFUL of delete-activity calls (ceil(1000 / 50) = 20), not one per token (1000), which is what + // bloated the orchestration history. The activity contract is one outcome per token, so the mock returns + // exactly that; the orchestrator asserts the count before zipping. + const int batchSize = 1000; + List tombstones = []; + for (int i = 0; i < batchSize; i++) + { + tombstones.Add(new LargePayloadTombstone( + PartitionId: 1, + InstanceKey: i, + PayloadId: i, + Token: $"blob:v2:https://acct.blob.core.windows.net/c/{i}", + Revision: 1)); + } + + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // Active on the first read, stopped on the second, so the perpetual loop runs exactly one cycle. + entities + .SetupSequence(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = batchSize }) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending }); + + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + + context + .Setup(c => c.CallActivityAsync>( + It.Is(n => n.Name == nameof(GetLargePayloadTombstonesActivity)), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(tombstones); + + int chunkActivityCalls = 0; + int tokensDeleted = 0; + context + .Setup(c => c.CallActivityAsync>( + It.Is(n => n.Name == nameof(DeleteExternalBlobActivity)), + It.IsAny(), + It.IsAny())) + .Returns((_, input, _) => + { + List chunk = (List)input!; + Interlocked.Increment(ref chunkActivityCalls); + Interlocked.Add(ref tokensDeleted, chunk.Count); + + // One outcome per token, positionally aligned - the shape the orchestrator asserts before zipping. + List outcomes = new(chunk.Count); + foreach (string _ in chunk) + { + outcomes.Add(new BlobPurgeOutcome(LargePayloadPurgeDisposition.Deleted)); + } + + return Task.FromResult(outcomes); + }); + + List? reported = null; + context + .Setup(c => c.CallActivityAsync( + It.Is(n => n.Name == nameof(ReportLargePayloadPurgeResultsActivity)), + It.IsAny(), + It.IsAny())) + .Callback((_, input, _) => reported = (List)input!) + .Returns(Task.CompletedTask); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: batchSize)); + + // Assert - 20 chunk activities for the whole batch, not 1000; every token was still handled exactly once; + // and one result is reported per row so the backend hears about all 1000. + this.AssertNoCycleFailed(); + chunkActivityCalls.Should().Be(20); + tokensDeleted.Should().Be(batchSize); + reported.Should().NotBeNull(); + reported!.Should().HaveCount(batchSize); + } + /// /// Guards against the whole test passing through the orchestrator's catch-all cycle handler, which would /// leave every observation empty and make the assertions vacuous. diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 94ad4830..893ef5fe 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -28,7 +28,7 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailed400_Quarantines() DeleteExternalBlobActivity activity = new(store, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; // Assert - quarantined (evidence preserved), never a success-shaped discard. The sanitized storage // error code is logged alongside the cause and is what distinguishes this from the parse failures. @@ -46,7 +46,7 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_RetriesAsTransien DeleteExternalBlobActivity activity = new(store, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); @@ -65,7 +65,7 @@ public async Task RunAsync_WhenDeleteThrowsAuthorizationFailure_Retries(int stat DeleteExternalBlobActivity activity = new(store, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); @@ -82,8 +82,8 @@ public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_Retries() DeleteExternalBlobActivity activity = new(store, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync( - null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); + BlobPurgeOutcome outcome = (await activity.RunAsync( + null!, ["blob:v2:https://other.blob.core.windows.net/c/abc123"]))[0]; // Assert - the cause is what separates this from the other retryable storage failures, which the // contract no longer distinguishes. @@ -101,7 +101,7 @@ public async Task RunAsync_V1Token_QuarantinesWithoutCallingStore() DeleteExternalBlobActivity activity = new(store.Object, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, ["blob:v1:payloads:abc123"]))[0]; // Assert - quarantined by the gate, and the store's DeleteAsync was never invoked. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); @@ -118,7 +118,7 @@ public async Task RunAsync_UnknownTokenVersion_RetriesWithoutCallingStore() DeleteExternalBlobActivity activity = new(store.Object, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v9:https://acct.blob.core.windows.net/c/x"); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, ["blob:v9:https://acct.blob.core.windows.net/c/x"]))[0]; // Assert - retried, NOT quarantined: quarantine is terminal and requires an operator to unwind, while a // deferral only leaves the row idle and visible. This branch and the token branches that quarantine are @@ -139,7 +139,7 @@ public async Task RunAsync_MalformedV2Token_Quarantines() DeleteExternalBlobActivity activity = new(store, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v2:not-a-uri"); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, ["blob:v2:not-a-uri"]))[0]; // Assert - contrast with the unknown-prefix case above, which is retried. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); @@ -157,7 +157,7 @@ public async Task RunAsync_V2Token_CallsStoreAndReportsDeleted() DeleteExternalBlobActivity activity = new(store.Object, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; // Assert - an ordinary success is silent; any log here would mean a failure branch was taken. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); @@ -176,7 +176,7 @@ public async Task RunAsync_WhenBlobAlreadyAbsent_ReportsDeleted() DeleteExternalBlobActivity activity = new(store.Object, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); @@ -194,7 +194,7 @@ public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeletin DeleteExternalBlobActivity activity = new(store.Object, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; // Assert - the tombstone is still resolved: a blob the store does not own is not the store's to delete, // and re-serving the row forever would never make it deletable. The reported result is now identical to @@ -212,7 +212,7 @@ public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombst DeleteExternalBlobActivity activity = new(store, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; // Assert - retried (tombstone preserved): resolving it would destroy the backend's cleanup ledger while // the blob survives. @@ -229,13 +229,95 @@ public async Task RunAsync_WhenDeleteTimesOut_RetriesAsTransient() DeleteExternalBlobActivity activity = new(store, logger); // Act - BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; // Assert - storage reported no code here, so the exception's type name is what identifies the failure. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnexpectedFailure:TimeoutException")); } + [Fact] + public async Task RunAsync_MixedChunk_AttributesEachOutcomeToItsOwnTokenByIndex() + { + // Arrange - one chunk whose five tokens resolve to deliberately different dispositions and complete in + // the REVERSE of their input order. Adjacent indices differ, so a driver that recorded results in + // completion order (or shifted by one) would misattribute at least one row. Index 3 throws a raw + // exception from the store to prove that a single failing token neither faults its peers nor escapes the + // activity. + string[] tokens = + [ + "blob:v2:https://acct.blob.core.windows.net/payloads/0", + "blob:v2:https://acct.blob.core.windows.net/payloads/1", + "blob:v2:https://acct.blob.core.windows.net/payloads/2", + "blob:v2:https://acct.blob.core.windows.net/payloads/3", + "blob:v2:https://acct.blob.core.windows.net/payloads/4", + ]; + + Dictionary> gates = new(); + foreach (string token in tokens) + { + // Synchronous continuations (the default) make completion strictly ordered: each Set* below resolves + // exactly one delete inline, in call order, so completion order is deterministically the reverse of + // input order. The index-based driver ignores completion order, but a regression to completion-order + // recording would then attribute deterministically wrong - which is what makes this test bite. + gates[token] = new TaskCompletionSource(); + } + + ControlledPayloadStore store = new(gates); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); + + // Act - start the chunk (every delete is now parked on its gate), then release the tokens back-to-front + // so completion order is the exact reverse of input order. + Task> run = activity.RunAsync(null!, [.. tokens]); + + gates[tokens[4]].SetResult(PayloadDeleteOutcome.AlreadyAbsent); // -> Deleted + gates[tokens[3]].SetException(new TimeoutException()); // -> Retry + gates[tokens[2]].SetResult(PayloadDeleteOutcome.Deleted); // -> Deleted + gates[tokens[1]].SetException(new RequestFailedException(503, "busy", "ServerBusy", null)); // -> Retry + gates[tokens[0]].SetException( + new RequestFailedException(400, "bad", "InvalidResourceName", null)); // -> Quarantined + + List outcomes = await run; + + // Assert - each disposition sits at its own token's index regardless of completion order, and the raw + // failure at index 3 left indices 0, 1, 2, and 4 untouched. + outcomes.Should().HaveCount(tokens.Length); + outcomes[0].Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + outcomes[1].Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcomes[2].Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + outcomes[3].Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcomes[4].Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + } + + [Fact] + public async Task RunAsync_WhenSingleDeleteExceedsTimeout_ReturnsRetryInsteadOfHanging() + { + // Arrange - a store whose delete never completes on its own and only observes cancellation. With the + // per-delete timeout the activity must abandon it and classify it retryable, rather than pinning the + // concurrency slot for the store's full multi-minute retry budget. + BlockingPayloadStore store = new(); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger) + { + SingleDeleteTimeout = TimeSpan.FromMilliseconds(50), + }; + + // Act - guard the await so a regression (an unbounded delete) surfaces as a bounded assertion failure + // instead of hanging the whole test run. + Task> run = activity.RunAsync(null!, [V2Token]); + Task first = await Task.WhenAny(run, Task.Delay(TimeSpan.FromSeconds(10))); + first.Should().BeSameAs(run, "the delete must yield to its timeout rather than run unbounded"); + + List outcomes = await run; + + // Assert - the timeout surfaced as a cancellation and was classified retryable by the catch-all, exactly + // as a network timeout would be; no exception escaped RunAsync. + outcomes.Should().ContainSingle(); + outcomes[0].Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnexpectedFailure:TaskCanceledException")); + } + sealed class StubPayloadStore : PayloadStore { readonly Exception? deleteError; @@ -255,4 +337,41 @@ public override Task DownloadAsync(string token, CancellationToken cance public override bool IsKnownPayloadToken(string value) => true; } + + sealed class ControlledPayloadStore : PayloadStore + { + readonly IReadOnlyDictionary> gates; + + public ControlledPayloadStore( + IReadOnlyDictionary> gates) => this.gates = gates; + + public override Task DeleteAsync(string token, CancellationToken cancellationToken) => + this.gates[token].Task; + + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => true; + } + + sealed class BlockingPayloadStore : PayloadStore + { + public override async Task DeleteAsync(string token, CancellationToken cancellationToken) + { + // Never completes on its own; only cancellation (the activity's per-delete timeout) ends the wait. + await Task.Delay(Timeout.Infinite, cancellationToken); + return PayloadDeleteOutcome.Deleted; + } + + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => true; + } } From 7833e11254a9a68640c51ffffa68d35a1d522c2e Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 14:29:41 -0400 Subject: [PATCH 39/47] Address three review items: version/soft-delete docs, shared limit constant, test null-forgiving Three small, independent review items; item 1 is documentation-only with zero behavior change. Item 1 (r3677836542 - blob versioning / soft delete): IncludeSnapshots deletes snapshots and the base blob but not blob *versions*, and soft delete retains bytes container-side regardless of how the delete is issued, so acknowledging a purge does not guarantee reclamation. Documented this rather than enumerating versions, because even deleting every version would not guarantee reclamation under a soft-delete/retention policy - immediate reclamation is unobtainable client-side and belongs to a lifecycle-management policy. Changes are doc-only: - PayloadDeleteOutcome.Deleted now states it means the delete was accepted by storage, not that bytes were reclaimed. - LargePayloadStorageOptions.AutoPurge XML doc gains a paragraph on versioning/soft delete and recommends a lifecycle policy. - An inline comment at BlobPayloadStore's DeleteIfExistsAsync call records the IncludeSnapshots-vs-versions distinction and why versions are not enumerated. Item 2 (r3767474415 - move the 1000 limit to a shared constant): the GetLargePayloadTombstones request limit was hard-coded as the literal 1000 in two validators, each with a duplicated message string, so the client and the auto-purge extension could silently drift. Introduced a single public authority LargePayloadTombstone.MaxRequestLimit = 1000 (in Client core, reachable by both call sites) and pointed both validators at it, including building the exception message from it. The extension's internal BlobPurgeConstants.MaxBatchSize now delegates to the same constant instead of redefining 1000, so there is one source of truth. Adds public API surface (the new public const); the repo has no public-API baseline file to update. Item 3 (r3772204948 - missing null-forgiving operator): BlobPurgeJobStarterTests line 82 accessed run.Signal.Value without the '!' that line 81 uses; added it for consistency. One character; no behavior change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- src/Client/Core/LargePayloadTombstone.cs | 11 ++++++++++- src/Client/Grpc/GrpcDurableTaskClient.cs | 4 ++-- .../Activities/GetLargePayloadTombstonesActivity.cs | 4 ++-- .../AutoPurge/Constants/BlobPurgeConstants.cs | 9 ++++++--- .../Options/LargePayloadStorageOptions.cs | 9 +++++++++ .../PayloadStore/BlobPayloadStore.cs | 7 +++++++ .../PayloadStore/PayloadDeleteOutcome.cs | 5 ++++- .../AutoPurge/BlobPurgeJobStarterTests.cs | 2 +- 8 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/Client/Core/LargePayloadTombstone.cs b/src/Client/Core/LargePayloadTombstone.cs index c11a40db..77316d4d 100644 --- a/src/Client/Core/LargePayloadTombstone.cs +++ b/src/Client/Core/LargePayloadTombstone.cs @@ -20,4 +20,13 @@ namespace Microsoft.DurableTask.Client; /// a per-row lease. /// public sealed record LargePayloadTombstone( - int PartitionId, long InstanceKey, long PayloadId, string Token, long Revision); + int PartitionId, long InstanceKey, long PayloadId, string Token, long Revision) +{ + /// + /// The maximum number of tombstones a single GetLargePayloadTombstones request may ask for. This is + /// the backend-imposed ceiling on the request limit - the backend rejects requests above it. Validators that + /// bound a caller-supplied limit compare against this shared value rather than a hard-coded literal so the + /// bound cannot drift between the client and the auto-purge extension. + /// + public const int MaxRequestLimit = 1000; +} diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 9a5fcb98..ddfb03ed 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -628,10 +628,10 @@ public override async Task> GetOrchestrationHistoryAsync( public override async Task> GetLargePayloadTombstonesAsync( int limit, CancellationToken cancellation = default) { - if (limit <= 0 || limit > 1000) + if (limit <= 0 || limit > LargePayloadTombstone.MaxRequestLimit) { throw new ArgumentOutOfRangeException( - nameof(limit), limit, "Limit must be greater than 0 and less than or equal to 1000."); + nameof(limit), limit, $"Limit must be greater than 0 and less than or equal to {LargePayloadTombstone.MaxRequestLimit}."); } P.GetLargePayloadTombstonesResponse response; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs index 0d9dc78c..2bded9b7 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs @@ -27,10 +27,10 @@ internal sealed class GetLargePayloadTombstonesActivity( /// public override async Task> RunAsync(TaskActivityContext context, int input) { - if (input <= 0 || input > 1000) + if (input <= 0 || input > LargePayloadTombstone.MaxRequestLimit) { throw new ArgumentOutOfRangeException( - nameof(input), input, "Limit must be greater than 0 and less than or equal to 1000."); + nameof(input), input, $"Limit must be greater than 0 and less than or equal to {LargePayloadTombstone.MaxRequestLimit}."); } P.GetLargePayloadTombstonesResponse response; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index 0138da35..a1afbc41 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Microsoft.DurableTask.Client; + namespace Microsoft.DurableTask.AzureBlobPayloads; /// @@ -21,10 +23,11 @@ static class BlobPurgeConstants public const int DefaultBatchSize = 500; /// - /// The maximum batch size the auto-purge job may request per cycle. Mirrors the gRPC - /// GetLargePayloadTombstones contract, which rejects limits greater than 1000. + /// The maximum batch size the auto-purge job may request per cycle. Delegates to + /// , the single authority for the gRPC + /// GetLargePayloadTombstones contract bound, so the two cannot drift. /// - public const int MaxBatchSize = 1000; + public const int MaxBatchSize = LargePayloadTombstone.MaxRequestLimit; /// /// The fixed instance ID of the client-to-entity bridge orchestration the starter schedules to ensure the diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 49cb020d..24dac184 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -125,12 +125,21 @@ public int ThresholdBytes /// Defaults to false (opt-in). /// /// + /// /// Auto-purge reclaims only blobs referenced by self-describing blob:v2: tokens. Payloads written by /// SDK versions that emitted legacy blob:v1: tokens are not reclaimed, because a v1 token identifies /// the container by name only and not the storage account, so the delete cannot be verified. Their backing /// blobs remain in storage exactly as they did before auto-purge existed - this is not a new leak introduced /// by auto-purge - and the backend removes their rows normally. Upgrading to an SDK version that writes v2 /// tokens makes newly-externalized payloads eligible. + /// + /// + /// When the payload container has blob versioning or soft delete enabled, auto-purge deletes the current + /// base blob, but retained previous versions and soft-deleted blobs continue to consume storage until a + /// lifecycle-management policy or the configured retention period reclaims them. No client-side delete can + /// guarantee immediate reclamation under those policies. Configure a lifecycle-management policy on the + /// payload container if immediate reclamation matters. + /// /// public bool AutoPurge { get; set; } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index bc597e4c..9c36b2b1 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -299,6 +299,13 @@ public override async Task DeleteAsync(string token, Cance // delete behaves as a single check-and-delete without taking a lease. // Idempotent by design: DeleteIfExistsAsync returns false (rather than throwing) when the blob is // already gone, so re-delivered tombstones and concurrent purges from multiple worker replicas are safe. + // IncludeSnapshots removes the blob's snapshots along with the base blob, but it does NOT delete blob + // *versions*: with versioning enabled, deleting the base blob turns the current version into a retained + // previous version. Versions are deliberately not enumerated and deleted here - doing so still would not + // guarantee the bytes are reclaimed, because soft delete is a container-level policy that retains deleted + // content for its retention period regardless of how the delete was issued. Immediate reclamation is + // therefore unobtainable client-side and belongs to a lifecycle-management policy or the retention + // expiry, so Deleted means "accepted by storage", not "bytes reclaimed" (see PayloadDeleteOutcome.Deleted). Response deleted = await blob.DeleteIfExistsAsync( DeleteSnapshotsOption.IncludeSnapshots, conditions: new BlobRequestConditions { IfMatch = properties.ETag }, diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs index 532cbcd9..e9a28d2b 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs @@ -10,7 +10,10 @@ namespace Microsoft.DurableTask; public enum PayloadDeleteOutcome { /// - /// The payload's backing object was deleted by this call. + /// The delete was accepted by storage on this call - the backing object's current version was removed. + /// This means the delete was accepted, not that the underlying bytes were reclaimed: if the container has + /// blob versioning or soft delete enabled, the prior version or the soft-deleted blob is retained until a + /// lifecycle-management policy or the configured retention period removes it. /// Deleted, diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index 83d51fb2..6453371b 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -79,7 +79,7 @@ public async Task StartAsync_WhenAutoPurgeDisabledAndJobIsActive_SignalsJobToSto // Assert run.Signal.Should().NotBeNull("the disable path must tell a running job to stop"); run.Signal!.Value.Id.Should().Be(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); - run.Signal.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + run.Signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); } [Fact] From 46a4246851055aa5a8f83fbaa7809cb5ead13322 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 15:04:39 -0400 Subject: [PATCH 40/47] Test named-builder, combined-host, empty-fetch, and continue-as-new purge paths Close the remaining auto-purge test gaps with tests only; no product code changes. - Named client and worker builders: resolve LargePayloadStorageOptions, the entity-support flip, the intercepted gRPC options, the starter/store and the purge activities under a non-empty builder name, plus a negative test proving configuration does not leak to other names or the default name. - Combined client + worker host: exactly one shared PayloadStore and one client-registered BlobPurgeJobStarter (not one per builder), with the purge activity still constructible. - Empty fetch: an Active job whose fetch returns nothing idles on the timer without invoking the delete or report activities. - Continue-as-new bound: the perpetual orchestrator runs exactly ContinueAsNewFrequency cycles before a single ContinueAsNew that resets the cycle count and carries the job identity and batch size forward. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- .../BlobPurgeJobOrchestratorTests.cs | 88 +++++++++++ .../UseExternalizedPayloadsTests.cs | 146 ++++++++++++++++++ 2 files changed, 234 insertions(+) diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs index 9f392ab2..95b2395e 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs @@ -226,6 +226,94 @@ public async Task RunAsync_DeletesLargeBatch_InChunksNotOneActivityPerToken() reported!.Should().HaveCount(batchSize); } + [Fact] + public async Task RunAsync_WhenFetchReturnsNoTombstones_IdlesWithoutDeletingOrReporting() + { + // Arrange - an Active job whose fetch comes back empty. Nothing is due, so the cycle must short-circuit + // to the idle timer and touch neither the delete activity nor the report activity: an empty backend + // response has to cost a wait and nothing else, or every quiet worker would hammer storage and the + // backend on every tick. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 250 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert - the fetch ran (recording the batch size) and the idle timer was created, but neither the + // delete nor the report activity was ever invoked on the empty batch. + this.AssertNoCycleFailed(); + this.requested.Should().Equal(250); + context.Verify( + c => c.CreateTimer(It.IsAny(), It.IsAny()), + Times.AtLeastOnce); + context.Verify( + c => c.CallActivityAsync>( + It.Is(n => n.Name == nameof(DeleteExternalBlobActivity)), + It.IsAny(), + It.IsAny()), + Times.Never); + context.Verify( + c => c.CallActivityAsync( + It.Is(n => n.Name == nameof(ReportLargePayloadPurgeResultsActivity)), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task RunAsync_ContinuesAsNewOnlyAfterConfiguredCycles() + { + // Arrange - a job that stays Active with nothing to purge, so the continue-as-new guard is the ONLY + // thing that can end the perpetual loop. That guard is the sole bound on history growth here: a fresh + // run must process exactly ContinueAsNewFrequency cycles and then continue-as-new with a reset count. + // Firing early would reset the job's progress window too often; never firing would let the orchestration + // history grow without bound until the instance died days later. + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + // Always Active: the job never stops, so continue-as-new is the only exit from the loop. + entities + .Setup(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 250 }); + + context + .Setup(c => c.CallActivityAsync>( + It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, input, _) => this.requested.Add((int)input!)) + .ReturnsAsync([]); + + BlobPurgeJobRunRequest? restarted = null; + context + .Setup(c => c.ContinueAsNew(It.IsAny(), It.IsAny())) + .Callback((input, _) => restarted = (BlobPurgeJobRunRequest)input!); + + // Act - start fresh (zero processed cycles). + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 250)); + + // Assert - five cycles ran (one empty fetch each) before a single continue-as-new that reset the cycle + // count while carrying the same job identity and batch size forward. The five recorded fetches are what + // prove it did not continue-as-new early; the single ContinueAsNew is what proves it does not grow the + // history forever. + this.AssertNoCycleFailed(); + this.requested.Should().HaveCount(5); + context.Verify(c => c.ContinueAsNew(It.IsAny(), It.IsAny()), Times.Once); + restarted.Should().NotBeNull(); + restarted!.JobEntityId.Should().Be(JobEntityId); + restarted.PurgeBatchSize.Should().Be(250); + restarted.ProcessedCycles.Should().Be(0); + } + /// /// Guards against the whole test passing through the orchestrator's catch-all cycle handler, which would /// leave every observation empty and make the assertions vacuous. diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index a0c9eb11..1c8f74a4 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -5,6 +5,7 @@ using Grpc.Core; using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Grpc; using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; @@ -229,4 +230,149 @@ public void UseExternalizedPayloads_WorkerOnly_PurgeActivitiesAreConstructible() constructGet.Should().NotThrow(); constructReport.Should().NotThrow(); } + + [Fact] + public void UseExternalizedPayloads_NamedClientBuilder_ResolvesEverythingUnderThatName() + { + // Arrange - every other test in this file drives builder.Name == string.Empty, so the entire named code + // path had no coverage: the storage options, the client's entity-support flip, the intercepted gRPC + // client options and the purge starter are ALL keyed on builder.Name. Drive a non-empty name end to end. + const string name = "client-hub"; + ServiceCollection services = new(); + services.AddSingleton>(NullLogger.Instance); + services.AddSingleton(Mock.Of()); + services.AddOptions(name) + .Configure(o => o.CallInvoker = Mock.Of()); + + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(name); + + // Act + builder.Object.UseExternalizedPayloads(options => + { + options.ConnectionString = "UseDevelopmentStorage=true"; + options.AutoPurge = true; + }); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - the storage options, the entity-support flip and the intercepted gRPC client options all + // materialize under the builder name, and the starter and store resolve from the built provider. + provider.GetRequiredService>().Get(name) + .AutoPurge.Should().BeTrue(); + provider.GetRequiredService>().Get(name) + .EnableEntitySupport.Should().BeTrue(); + provider.GetRequiredService>().Get(name) + .CallInvoker.Should().NotBeNull(); + provider.GetServices().OfType().Should().ContainSingle(); + provider.GetRequiredService().Should().BeOfType(); + } + + [Fact] + public void UseExternalizedPayloads_NamedWorkerBuilder_ResolvesEverythingUnderThatName() + { + // Arrange - the worker mirror of the named-client gap: the storage options, the worker's entity-support + // flip, the resolved auto-purge handshake flag and the sidecar client the activities inject are all keyed + // on builder.Name, and every existing worker test uses the empty name. Drive a non-empty name. + const string name = "worker-hub"; + ServiceCollection services = new(); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddOptions(name) + .Configure(o => o.CallInvoker = Mock.Of()); + + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(name); + + // Act + builder.Object.UseExternalizedPayloads(options => + { + options.ConnectionString = "UseDevelopmentStorage=true"; + options.AutoPurge = true; + }); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - all three option sets resolve under the name, the store is blob-backed, and both purge + // activities construct through the exact reflection path DurableTaskRegistry uses at dispatch. + provider.GetRequiredService>().Get(name) + .AutoPurge.Should().BeTrue(); + provider.GetRequiredService>().Get(name) + .EnableEntitySupport.Should().BeTrue(); + provider.GetRequiredService>().Get(name) + .LargePayloadAutoPurgeEnabled.Should().BeTrue(); + provider.GetRequiredService().Should().BeOfType(); + + Action constructGet = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(GetLargePayloadTombstonesActivity)); + Action constructReport = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(ReportLargePayloadPurgeResultsActivity)); + constructGet.Should().NotThrow(); + constructReport.Should().NotThrow(); + } + + [Fact] + public void UseExternalizedPayloads_NamedClientBuilder_DoesNotLeakOptionsToOtherNames() + { + // Arrange - configure ONLY the "client-hub" name. This is the assertion with teeth: a happy-path named + // test still passes even if every .Get(builder.Name) were hard-coded to Options.DefaultName, because the + // requested name and the default would resolve to the same populated instance. Asserting that OTHER names + // stay at their defaults is what actually pins the configuration to builder.Name. + const string name = "client-hub"; + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(name); + + // Act + builder.Object.UseExternalizedPayloads(options => + { + options.ConnectionString = "UseDevelopmentStorage=true"; + options.AutoPurge = true; + }); + using ServiceProvider provider = services.BuildServiceProvider(); + IOptionsMonitor monitor = + provider.GetRequiredService>(); + + // Assert - neither a different name nor the default name observes "client-hub"'s configuration. + monitor.Get("other-hub").AutoPurge.Should().BeFalse(); + monitor.Get("other-hub").ConnectionString.Should().NotBe("UseDevelopmentStorage=true"); + monitor.Get(string.Empty).AutoPurge.Should().BeFalse(); + monitor.Get(string.Empty).ConnectionString.Should().NotBe("UseDevelopmentStorage=true"); + } + + [Fact] + public async Task UseExternalizedPayloads_ClientAndWorkerInOneHost_ShareOneStoreAndOneStarter() + { + // Arrange - the combined "client triggers, worker executes" host: both AddDurableTaskClient and + // AddDurableTaskWorker call UseExternalizedPayloads in one ServiceCollection. PayloadStore is registered + // with TryAdd on both sides, so the container must hold exactly one; and only the client registers the + // purge starter, so there must be exactly one of those too - not one per builder. + ServiceCollection services = new(); + services.AddSingleton(NullLoggerFactory.Instance); + services.AddSingleton(typeof(ILogger<>), typeof(NullLogger<>)); + services.AddDurableTaskClient(builder => + { + builder.UseGrpc(options => options.CallInvoker = Mock.Of()); + builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + }); + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(options => options.CallInvoker = Mock.Of()); + builder.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + }); + + // AddDurableTaskClient registers a ClientContainer that is IAsyncDisposable-only, so the provider must be + // disposed asynchronously. + await using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - one shared store, one starter (client-registered), and the worker's purge activity still + // constructs in the combined container. + provider.GetServices().Should().ContainSingle() + .Which.Should().BeOfType(); + provider.GetServices().OfType().Should().ContainSingle(); + + Action constructActivity = () => + ActivatorUtilities.GetServiceOrCreateInstance(provider, typeof(GetLargePayloadTombstonesActivity)); + constructActivity.Should().NotThrow(); + } } From 0857bdeb476a736b38860bc3b00615ca4fa4c93c Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 15:14:02 -0400 Subject: [PATCH 41/47] docs: correct blob soft delete/versioning to storage-account-level scope Blob soft delete and blob versioning are storage-account-level settings (Blob service properties), not container-level. Correct the three doc/comment sites added for the auto-purge storage-semantics note so customers reading the public XML docs are not misinformed: - LargePayloadStorageOptions.AutoPurge - PayloadDeleteOutcome.Deleted - BlobPayloadStore delete comment above DeleteIfExistsAsync The correction strengthens the existing argument: because these are account-level policies, a single container cannot opt out, so retained versions / soft-deleted bytes are even further outside the SDK's control. Also clarify that reclamation is governed by an account lifecycle-management policy whose rule can be scoped to the payload container's blob prefix. Comments / XML docs only; no code, signature, or behavior change. IncludeSnapshots is unchanged and version enumeration remains deliberately not implemented. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- .../Options/LargePayloadStorageOptions.cs | 11 ++++++----- .../PayloadStore/BlobPayloadStore.cs | 10 ++++++---- .../PayloadStore/PayloadDeleteOutcome.cs | 6 +++--- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 24dac184..ffaad7dd 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -134,11 +134,12 @@ public int ThresholdBytes /// tokens makes newly-externalized payloads eligible. /// /// - /// When the payload container has blob versioning or soft delete enabled, auto-purge deletes the current - /// base blob, but retained previous versions and soft-deleted blobs continue to consume storage until a - /// lifecycle-management policy or the configured retention period reclaims them. No client-side delete can - /// guarantee immediate reclamation under those policies. Configure a lifecycle-management policy on the - /// payload container if immediate reclamation matters. + /// When the storage account holding the payload container has blob versioning or blob soft delete enabled, + /// auto-purge deletes the current base blob, but retained previous versions and soft-deleted blobs continue + /// to consume storage until a lifecycle-management policy or the configured retention period reclaims them. + /// Both settings are account-level, so a single container cannot opt out of them, and no client-side delete + /// can guarantee immediate reclamation under those policies. Configure an account lifecycle-management policy + /// with a rule scoped to the payload container's blob prefix if immediate reclamation matters. /// /// public bool AutoPurge { get; set; } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index 9c36b2b1..0601751d 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -302,10 +302,12 @@ public override async Task DeleteAsync(string token, Cance // IncludeSnapshots removes the blob's snapshots along with the base blob, but it does NOT delete blob // *versions*: with versioning enabled, deleting the base blob turns the current version into a retained // previous version. Versions are deliberately not enumerated and deleted here - doing so still would not - // guarantee the bytes are reclaimed, because soft delete is a container-level policy that retains deleted - // content for its retention period regardless of how the delete was issued. Immediate reclamation is - // therefore unobtainable client-side and belongs to a lifecycle-management policy or the retention - // expiry, so Deleted means "accepted by storage", not "bytes reclaimed" (see PayloadDeleteOutcome.Deleted). + // guarantee the bytes are reclaimed, because blob soft delete is a storage-account-level policy that + // retains deleted content for its retention period regardless of how the delete was issued and regardless + // of which container the blob lives in. Immediate reclamation is therefore unobtainable client-side and + // belongs to an account lifecycle-management policy (whose rule can be scoped to the payload container's + // blob prefix) or the retention expiry, so Deleted means "accepted by storage", not "bytes reclaimed" + // (see PayloadDeleteOutcome.Deleted). Response deleted = await blob.DeleteIfExistsAsync( DeleteSnapshotsOption.IncludeSnapshots, conditions: new BlobRequestConditions { IfMatch = properties.ETag }, diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs index e9a28d2b..afc95115 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs @@ -11,9 +11,9 @@ public enum PayloadDeleteOutcome { /// /// The delete was accepted by storage on this call - the backing object's current version was removed. - /// This means the delete was accepted, not that the underlying bytes were reclaimed: if the container has - /// blob versioning or soft delete enabled, the prior version or the soft-deleted blob is retained until a - /// lifecycle-management policy or the configured retention period removes it. + /// This means the delete was accepted, not that the underlying bytes were reclaimed: if the storage account + /// has blob versioning or blob soft delete enabled, the prior version or the soft-deleted blob is retained + /// until a lifecycle-management policy or the configured retention period removes it. /// Deleted, From 720368e451c213ec17d4fbbc259ddf6c45e1ec12 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 15:19:15 -0400 Subject: [PATCH 42/47] docs: document delete guarantee on base PayloadStore contract The virtual PayloadStore.DeleteAsync already binds third-party stores to idempotency and ownership (NotStoreOwned), but its says only "whether the object was deleted" - silent on what a successful delete actually guarantees, which is exactly the ambiguity flagged in review. Add product-neutral contract prose to the : a Deleted outcome means the store no longer references the object and the underlying storage accepted the delete, NOT that the bytes were reclaimed; storage-level retention features such as versioning, soft delete, or retention policies may keep the content for a policy-defined period, and implementations are not expected to defeat them. Store-agnostic by design - PayloadStore is a public extensibility point - with no Azure/blob/container specifics, mirroring the product-neutral NotStoreOwned naming. The Azure-specific reclamation semantics remain documented on the concrete BlobPayloadStore path (inline comment, PayloadDeleteOutcome.Deleted, AutoPurge remarks), so this adds the missing base-contract sentence without duplicating them. Docs only; no code, signature, or behavior change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs index ee6f22b2..92636789 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs @@ -35,6 +35,10 @@ public abstract class PayloadStore /// Implementations must delete only objects they created; an object that carries no proof of the /// store's ownership must be left untouched and reported as /// . + /// A result means the store no longer references the object + /// and the underlying storage accepted the delete; it does not guarantee that the bytes have been + /// reclaimed. Storage-level retention features such as versioning, soft delete, or retention policies + /// may keep the content for a policy-defined period, and implementations are not expected to defeat them. /// /// The opaque reference token. /// Cancellation token. From 0034bb00ec158d280adebeffba2e8f61b39928d2 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 17:33:54 -0400 Subject: [PATCH 43/47] Document per-host single-configuration constraint for externalized payloads Externalized payloads are configured per host, not per named builder: the PayloadStore and the purge TaskHubSidecarServiceClient are registered as container-wide singletons, so the first builder that calls UseExternalizedPayloads supplies the configuration the whole host shares. Configuring multiple named workers/clients in one host with different storage accounts or different backends is therefore not supported - later builders silently reuse the first registration. Adds class-level to both the worker and client DurableTask*BuilderExtensionsAzureBlobPayloads classes stating the constraint, and extends the inline comment above the worker's sidecar-client TryAddSingleton to explain why a keyed/named registration cannot work here: the consuming purge activities are constructed from the plain IServiceProvider with no worker name in scope, so a keyed client would have no resolvable consumer. Comments and XML docs only - no behavior, options, or DI registration change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- ...askClientBuilderExtensions.AzureBlobPayloads.cs | 8 ++++++++ ...askWorkerBuilderExtensions.AzureBlobPayloads.cs | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index ed172b0f..099a5572 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -18,6 +18,14 @@ namespace Microsoft.DurableTask; /// /// Extension methods to enable externalized payloads using Azure Blob Storage for Durable Task Client. /// +/// +/// Externalized payloads are configured per host, not per named builder. The PayloadStore is registered +/// as a container-wide singleton (shared with the worker builder in the same host), so the first builder that +/// calls UseExternalizedPayloads supplies the configuration the whole host uses. Configuring multiple +/// named clients in the same host with different storage accounts or different backends is therefore not +/// supported: later builders silently share the first builder's registration. A single named client, or +/// several named clients that share one configuration, is fully supported. +/// public static class DurableTaskClientBuilderExtensionsAzureBlobPayloads { /// diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index 38d73386..46b633be 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -17,6 +17,14 @@ namespace Microsoft.DurableTask; /// /// Extension methods to enable externalized payloads using Azure Blob Storage for Durable Task Worker. /// +/// +/// Externalized payloads are configured per host, not per named builder. The PayloadStore and the +/// purge TaskHubSidecarServiceClient are registered as container-wide singletons, so the first builder +/// in the host that calls UseExternalizedPayloads supplies the configuration that both of them use. +/// Configuring multiple named workers in the same host with different storage accounts or different backends +/// is therefore not supported: later builders silently share the first builder's registration. A single named +/// worker, or several named workers that share one configuration, is fully supported. +/// public static class DurableTaskWorkerBuilderExtensionsAzureBlobPayloads { /// @@ -107,6 +115,12 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB // The purge activities talk to the backend over the worker's OWN transport, so a worker-only host // (which never registers a DurableTaskClient) can still run the job. This resolves the same CallInvoker // the worker itself uses, so the RPCs ride its channel and interceptor - no second connection. + // TryAddSingleton (rather than a keyed/named registration) is deliberate: the consumers - + // GetLargePayloadTombstonesActivity and ReportLargePayloadPurgeResultsActivity - are constructed from + // the plain IServiceProvider at dispatch with no worker name in scope, so a keyed registration would + // have no resolvable consumer. This is the per-host single-configuration constraint documented on the + // class remarks: in a multi-named-worker host the first builder's options win here. Do not "fix" this + // into keyed DI - without a worker-name-aware consumer there is nothing to resolve the keyed client. builder.Services.TryAddSingleton(sp => { GrpcDurableTaskWorkerOptions options = From a814ffc263ecb6e4b3687a5d91d98a27fa7a7e73 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 18:59:46 -0400 Subject: [PATCH 44/47] Quarantine unowned blobs, keep the disable path reconciling, drop client-facing purge APIs F1: DeleteExternalBlobActivity now quarantines a blob that carries no ownership marker (PayloadDeleteOutcome.NotStoreOwned) instead of reporting it Deleted. The ownership marker is new in this PR while main has been emitting v2 tokens without writing it, so on upgrade every pre-existing v2 blob reaches this path; reporting Deleted would make the backend hard-delete the tombstone row, orphaning the blob permanently and destroying the only durable record of it. Quarantine preserves the row and its token backend-side as evidence, matching the v1 and malformed-token branches. Removes the now-unused BlobPurgeBlobNotStoreOwned log (EventId 821). F2: BlobPurgeJobStarter.SignalJobStopAsync now reconciles on a fixed interval like the enabled path instead of signalling Stop once and returning. AutoPurge is per-host config, so during a rolling deployment a replica still on the enabled path re-creates the job every interval; a disabled replica that stopped it once would let it come straight back and the fleet would never converge. The loop is governed by the same cancellationToken as the enabled path, so shutdown ends it, and each pass reads state first so a job that is already stopped is a no-op read. F3: removes the client-facing GetLargePayloadTombstonesAsync and ReportLargePayloadPurgeResultsAsync from DurableTaskClient and their GrpcDurableTaskClient overrides. The purge activities inject the protobuf TaskHubSidecarServiceClient directly and carry their own enum mapping, so these virtuals were a duplicate of the product path, not the product path. The LargePayloadTombstone, LargePayloadPurgeResult, and LargePayloadPurgeDisposition contract types are kept. Deletes the duplicate LargePayloadPurgeUnimplementedTests (the same Unimplemented to NotImplementedException behavior is covered on the actual activity path by PurgeActivityUnimplementedTests) and repoints the enum-parity test at ReportLargePayloadPurgeResultsActivity. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- src/Client/Core/DurableTaskClient.cs | 24 ----- src/Client/Grpc/GrpcDurableTaskClient.cs | 84 ----------------- .../Activities/DeleteExternalBlobActivity.cs | 17 ++-- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 68 +++++++++----- .../AzureBlobPayloads/AutoPurge/Logs.cs | 3 - .../LargePayloadPurgeEnumParityTests.cs | 4 +- .../LargePayloadPurgeUnimplementedTests.cs | 92 ------------------- .../AutoPurge/BlobPurgeJobStarterTests.cs | 55 +++++++++++ .../DeleteExternalBlobActivityTests.cs | 13 +-- 9 files changed, 119 insertions(+), 241 deletions(-) delete mode 100644 test/Client/Grpc.Tests/LargePayloadPurgeUnimplementedTests.cs diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index c903574a..03303800 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -549,30 +549,6 @@ public virtual Task> ListInstanceIdsAsync( $"{this.GetType()} does not support listing orchestration instance IDs filtered by completed time."); } - /// - /// Gets a batch of due large-payload tombstones whose backing blobs a credentialed caller must delete. - /// - /// The maximum number of tombstones to request. - /// The cancellation token. - /// The batch of tombstones whose blobs should be deleted. - /// Thrown if this implementation does not support the operation. - public virtual Task> GetLargePayloadTombstonesAsync( - int limit, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support retrieving large-payload tombstones."); - - /// - /// Reports the outcome of each attempted large-payload blob deletion so the backend can resolve, - /// reschedule, or quarantine the corresponding tombstones. Every attempted row is reported, not just the - /// successful ones; the backend owns retry scheduling. - /// - /// The per-row outcomes of the attempted deletions. - /// The cancellation token. - /// A task that completes when the outcomes have been recorded. - /// Thrown if this implementation does not support the operation. - public virtual Task ReportLargePayloadPurgeResultsAsync( - IEnumerable results, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support reporting large-payload purge results."); - // TODO: Create task hub // TODO: Delete task hub diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index ddfb03ed..23350d4c 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -624,90 +624,6 @@ public override async Task> GetOrchestrationHistoryAsync( } } - /// - public override async Task> GetLargePayloadTombstonesAsync( - int limit, CancellationToken cancellation = default) - { - if (limit <= 0 || limit > LargePayloadTombstone.MaxRequestLimit) - { - throw new ArgumentOutOfRangeException( - nameof(limit), limit, $"Limit must be greater than 0 and less than or equal to {LargePayloadTombstone.MaxRequestLimit}."); - } - - P.GetLargePayloadTombstonesResponse response; - try - { - response = await this.sidecarClient.GetLargePayloadTombstonesAsync( - new P.GetLargePayloadTombstonesRequest { Limit = limit }, - cancellationToken: cancellation); - } - catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) - { - throw new NotImplementedException(e.Status.Detail); - } - catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) - { - throw new OperationCanceledException( - $"The {nameof(this.GetLargePayloadTombstonesAsync)} operation was canceled.", e, cancellation); - } - - List result = new(response.Tombstones.Count); - foreach (P.LargePayloadTombstone tombstone in response.Tombstones) - { - result.Add(new LargePayloadTombstone( - tombstone.PartitionId, - tombstone.InstanceKey, - tombstone.PayloadId, - tombstone.Token, - tombstone.Revision)); - } - - return result; - } - - /// - public override async Task ReportLargePayloadPurgeResultsAsync( - IEnumerable results, CancellationToken cancellation = default) - { - Check.NotNull(results); - - P.ReportLargePayloadPurgeResultsRequest request = new(); - foreach (LargePayloadPurgeResult result in results) - { - request.Results.Add(new P.LargePayloadPurgeResult - { - PartitionId = result.PartitionId, - InstanceKey = result.InstanceKey, - PayloadId = result.PayloadId, - Revision = result.Revision, - - // The managed disposition enum declares the same numeric values as its protobuf counterpart, - // so it maps across by value. This is the only enum on the message and it only travels - // outbound, so the SDK can never receive a value it does not know. - Disposition = (P.LargePayloadPurgeDisposition)result.Disposition, - }); - } - - if (request.Results.Count == 0) - { - return; - } - - try - { - await this.sidecarClient.ReportLargePayloadPurgeResultsAsync(request, cancellationToken: cancellation); - } - catch (RpcException e) when (e.StatusCode == StatusCode.Unimplemented) - { - throw new NotImplementedException(e.Status.Detail); - } - catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) - { - throw new OperationCanceledException( - $"The {nameof(this.ReportLargePayloadPurgeResultsAsync)} operation was canceled.", e, cancellation); - } - } - static AsyncDisposable GetCallInvoker(GrpcDurableTaskClientOptions options, ILogger logger, out CallInvoker callInvoker) { Func>? recreator = options.Internal.ChannelRecreator; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 7085e1ad..6be38f13 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -167,16 +167,21 @@ async Task DeleteAsync(string token) using CancellationTokenSource timeout = new(this.SingleDeleteTimeout); PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, timeout.Token); - // The blob exists but this store never wrote it, so it was left untouched. That is an expected - // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it would - // fill the quarantine set with non-defects. The tombstone is still resolved, because a blob the - // store does not own is not the store's to delete. + // The blob exists but this store never wrote it - it carries no ownership marker - so it is + // quarantined, not reported as deleted. The marker is new in this PR, but main has been emitting v2 + // tokens without writing it, so on upgrade every pre-existing v2 blob reaches this branch. Reporting + // Deleted here would tell the backend to hard-delete the tombstone row, orphaning that blob + // permanently AND destroying the only durable record that it exists. Quarantine keeps the row and its + // token backend-side as evidence and stops the backend polling it - the same disposition, and for the + // same reason, as the v1 and malformed-token branches: a delete that cannot be verified as this + // store's own must not be discarded as a success. if (outcome == PayloadDeleteOutcome.NotStoreOwned) { - this.logger.BlobPurgeBlobNotStoreOwned(); + this.logger.BlobPurgeDeleteQuarantined("NotStoreOwned", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); } - // Deleted, AlreadyAbsent, and NotStoreOwned are all terminal successes: none can be improved by + // Deleted and AlreadyAbsent are terminal successes: the blob is gone, and neither can be improved by // trying again. return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Deleted); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index a894491c..8789cc5f 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -17,7 +17,9 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, stops a /// running job when it is disabled, and no-ops with an error log when the registered store cannot delete. It /// never blocks host startup - the work runs on a background task that retries until the backend is reachable -/// and then, on the enabled path, keeps reconciling on a fixed interval for the lifetime of the process. +/// and then keeps reconciling on a fixed interval for the lifetime of the process. Both paths reconcile: the +/// disabled path must also repeat, because AutoPurge is per-host config and a peer still on the enabled path +/// keeps re-creating the job, so a disabled host that signalled Stop only once would never converge the fleet. /// The job is a per-task-hub singleton, so racing client processes converge on the same result. /// sealed class BlobPurgeJobStarter : IHostedService, IDisposable @@ -97,7 +99,9 @@ public Task StartAsync(CancellationToken cancellationToken) { // Turning auto-purge off has to reach the backend to mean anything. The job is a perpetual // orchestrator owned by the task hub, not by this process, so a job created while the flag was on - // keeps deleting blobs forever no matter how many hosts start with it off. Stop a running one. + // keeps deleting blobs forever no matter how many hosts start with it off. Stop a running one - and + // keep reconciling, so a peer still on the enabled path cannot quietly re-create it and leave the + // fleet non-convergent; see SignalJobStopAsync. // // The store gate below deliberately does NOT apply to this path. Stopping a job requires no ability // to delete anything, and a user who has switched to a store that cannot delete is precisely @@ -317,6 +321,8 @@ async Task SignalJobStopAsync(CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { + TimeSpan delay; + try { // Resolved inside this loop's try, not in StartAsync's body. A builder name that matches no @@ -381,23 +387,33 @@ async Task SignalJobStopAsync(CancellationToken cancellationToken) this.logger.BlobPurgeJobStateUnknown(ex); } - if (!stopNeeded) + if (stopNeeded) { - return; + // Signalled rather than driven through the bridge orchestration: unlike Create, Stop needs no + // fixed-instance dedupe. It is idempotent in the entity, so concurrent hosts converge on the + // same state and a repeat costs nothing beyond the signal itself. + await entities.SignalEntityAsync( + this.entityId, + nameof(BlobPurgeJob.Stop), + null, + null, + cancellationToken); + + this.logger.BlobPurgeJobStopRequested(); } - // Signalled rather than driven through the bridge orchestration: unlike Create, Stop needs no - // fixed-instance dedupe. It is idempotent in the entity, so concurrent hosts converge on the - // same state and a repeat costs nothing beyond the signal itself. - await entities.SignalEntityAsync( - this.entityId, - nameof(BlobPurgeJob.Stop), - null, - null, - cancellationToken); - - this.logger.BlobPurgeJobStopRequested(); - return; + // Reconcile on a fixed interval rather than returning after one pass, mirroring the enabled path. + // AutoPurge is per-host config, so during a rolling deployment a replica still on the enabled path + // re-issues Create every ReconcileInterval and pulls the job back to Active; a disabled replica + // that signalled Stop once and returned would let that happen unopposed, and after the whole fleet + // is configured off the job could still be Active - it would never converge. Repeating the pass + // means that once nothing re-creates the job, the next read here finds it not Active, logs + // BlobPurgeJobNotRunning and signals nothing. This does not spin on shutdown: the loop is governed + // by cancellationToken exactly like the enabled path, so StopAsync/Dispose end it. Steady-state + // cost is one read-only GetEntityAsync per interval - strictly cheaper than the enabled path, which + // also schedules an orchestration - and BlobPurgeJob.Stop's guard absorbs a repeat signal as a + // no-op that does not touch LastModifiedAt. + delay = this.ReconcileInterval; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -406,14 +422,18 @@ await entities.SignalEntityAsync( catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { this.logger.BlobPurgeStarterRetry(ex); - try - { - await Task.Delay(RetryDelay, cancellationToken); - } - catch (OperationCanceledException) - { - return; - } + delay = RetryDelay; + } + + try + { + await Task.Delay(delay, cancellationToken); + } + catch (OperationCanceledException) + { + // Shutdown. This is the only thing that ends the loop on the success path, which is why + // StopAsync can cancel a pass that is parked here instead of waiting out the interval. + return; } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 66aadd15..2499452f 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -43,9 +43,6 @@ static partial class Logs [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); - [LoggerMessage(EventId = 821, Level = LogLevel.Warning, Message = "An externalized payload blob does not carry this store's ownership marker, so it was left untouched; the tombstone is still resolved. This is expected for payloads written before the marker shipped, and for blobs the store never created whose token text matches the payload token grammar.")] - public static partial void BlobPurgeBlobNotStoreOwned(this ILogger logger); - [LoggerMessage(EventId = 822, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is already stopped; ignoring the stop request.")] public static partial void BlobPurgeJobAlreadyStopped(this ILogger logger, string? jobId); diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs index 48658410..fe1f9ce1 100644 --- a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs +++ b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs @@ -8,7 +8,7 @@ namespace Microsoft.DurableTask.Client.Grpc.Tests; /// -/// maps the managed purge disposition +/// ReportLargePayloadPurgeResultsActivity maps the managed purge disposition /// onto its protobuf counterpart by numeric value rather than by name, which is only correct while the two /// sides agree on every value. A silent drift would not fail to compile; it would send the backend a different /// disposition than the worker decided and delete or quarantine the wrong rows. These tests pin the mapping. @@ -57,7 +57,7 @@ public void InboundTypes_ExposeNoEnumMembers(Type inboundType) // Assert enumMembers.Should().BeEmpty( "an enum on an inbound type invalidates the numeric enum cast in " + - "GrpcDurableTaskClient.ReportLargePayloadPurgeResultsAsync. The SDK would map a value chosen by the " + + "ReportLargePayloadPurgeResultsActivity. The SDK would map a value chosen by the " + "backend - including one a newer backend added that this SDK does not know - onto a managed member " + "by raw numeric value, silently mis-dispositioning rows. Map inbound enums explicitly instead, with " + "a switch that handles unknown values"); diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeUnimplementedTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeUnimplementedTests.cs deleted file mode 100644 index dc36da7b..00000000 --- a/test/Client/Grpc.Tests/LargePayloadPurgeUnimplementedTests.cs +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Grpc.Core; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DurableTask.Client.Grpc.Tests; - -/// -/// Public-API parity for the mixed-rollout mapping: the two large-payload purge overrides on -/// must translate a gRPC Unimplemented from an older backend into a -/// , matching the precedent already used elsewhere in the client for -/// unsupported RPCs. -/// -public class LargePayloadPurgeUnimplementedTests -{ - readonly Mock loggerMock = new(); - - [Fact] - public async Task GetLargePayloadTombstonesAsync_WhenBackendUnimplemented_ThrowsNotImplemented() - { - // Arrange - the backend does not implement the fetch RPC. - GrpcDurableTaskClient client = this.CreateThrowingClient( - new RpcException(new Status(StatusCode.Unimplemented, "unknown method"))); - - // Act - Func act = () => client.GetLargePayloadTombstonesAsync(100); - - // Assert - await act.Should().ThrowAsync(); - } - - [Fact] - public async Task ReportLargePayloadPurgeResultsAsync_WhenBackendUnimplemented_ThrowsNotImplemented() - { - // Arrange - a non-empty result list is required to reach the RPC past the empty-request short-circuit. - GrpcDurableTaskClient client = this.CreateThrowingClient( - new RpcException(new Status(StatusCode.Unimplemented, "unknown method"))); - LargePayloadPurgeResult[] results = - { - new(1, 2, 3, 4, LargePayloadPurgeDisposition.Deleted), - }; - - // Act - Func act = () => client.ReportLargePayloadPurgeResultsAsync(results); - - // Assert - await act.Should().ThrowAsync(); - } - - GrpcDurableTaskClient CreateThrowingClient(RpcException error) - { - GrpcDurableTaskClientOptions options = new() - { - CallInvoker = new ThrowingCallInvoker(error), - }; - - return new GrpcDurableTaskClient("test", options, this.loggerMock.Object); - } - - /// - /// A minimal whose unary calls fault with a configured , - /// so the client's real RPC path runs and hits its own catch clause exactly as a live channel would. - /// - sealed class ThrowingCallInvoker(RpcException error) : CallInvoker - { - public override AsyncUnaryCall AsyncUnaryCall( - Method method, string? host, CallOptions options, TRequest request) - => new( - Task.FromException(error), - Task.FromResult(new Metadata()), - () => error.Status, - () => new Metadata(), - () => { }); - - public override TResponse BlockingUnaryCall( - Method method, string? host, CallOptions options, TRequest request) - => throw error; - - public override AsyncServerStreamingCall AsyncServerStreamingCall( - Method method, string? host, CallOptions options, TRequest request) - => throw new NotSupportedException(); - - public override AsyncClientStreamingCall AsyncClientStreamingCall( - Method method, string? host, CallOptions options) - => throw new NotSupportedException(); - - public override AsyncDuplexStreamingCall AsyncDuplexStreamingCall( - Method method, string? host, CallOptions options) - => throw new NotSupportedException(); - } -} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index 6453371b..54be1bd0 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -171,6 +171,61 @@ public async Task StartAsync_WhenAutoPurgeDisabledAndStoreCannotDelete_StillSign run.Signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); } + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabled_KeepsReconciling() + { + // Arrange - the disable path used to signal once and return, which does not converge during a rolling + // deployment: AutoPurge is per-host config, so a peer still on the enabled path re-creates the job every + // interval, and a disabled host that stopped it a single time would let it come straight back. The + // disable path has to keep reconciling on the same interval, exactly like the enabled path. A short + // interval rather than a fake clock; the test gates on the second read happening, never on time passing. + Mock entities = new("test"); + TaskCompletionSource secondRead = new(); + int reads = 0; + entities + .Setup(e => e.GetEntityAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => + { + if (Interlocked.Increment(ref reads) >= 2) + { + secondRead.TrySetResult(true); + } + + return Task.FromResult?>(MetadataFor(BlobPurgeJobStatus.Active)); + }); + entities + .Setup(e => e.SignalEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns(Task.CompletedTask); + + Mock client = new("test"); + client.Setup(c => c.Entities).Returns(entities.Object); + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(client.Object); + + BlobPurgeJobStarter starter = new( + provider.Object, + new BlobPayloadStore(new LargePayloadStorageOptions("UseDevelopmentStorage=true")), + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = false }), + "test", + new TestLogger()); + starter.ReconcileInterval = TimeSpan.FromMilliseconds(10); + + // Act + await starter.StartAsync(CancellationToken.None); + Task completed = await Task.WhenAny(secondRead.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await starter.StopAsync(CancellationToken.None); + + // Assert - a loop that returned after the first pass leaves this waiting until the timeout fires. + completed.Should().BeSameAs( + secondRead.Task, "the disable path must keep reconciling rather than stopping after one pass"); + } + [Fact] public async Task EnsureJob_SchedulesBridge_DedupingOnlyPendingAndRunning() { diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 893ef5fe..816caf98 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -184,7 +184,7 @@ public async Task RunAsync_WhenBlobAlreadyAbsent_ReportsDeleted() } [Fact] - public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeleting() + public async Task RunAsync_WhenBlobNotStoreOwned_Quarantines() { // Arrange - the blob exists but carries no ownership marker, so the store left it untouched. Mock store = new(); @@ -196,11 +196,12 @@ public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeletin // Act BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; - // Assert - the tombstone is still resolved: a blob the store does not own is not the store's to delete, - // and re-serving the row forever would never make it deletable. The reported result is now identical to - // an ordinary delete, so the log is the only thing that records the difference. - outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); - logger.Logs.Should().ContainSingle(l => l.Message.Contains("ownership marker")); + // Assert - quarantined, not discarded as a success. The ownership marker is new in this PR, so every v2 + // blob written before it shipped reaches this branch on upgrade; reporting Deleted would drop the + // tombstone row and orphan the blob permanently. Quarantine preserves the row and its token as evidence, + // and the logged cause is the only record of why the row reached that disposition. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("NotStoreOwned")); } [Fact] From 9be9931b5277d9132723cb855da36271afc4fbfe Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 19:05:33 -0400 Subject: [PATCH 45/47] Reword disabled-path test helper comment for the perpetual reconcile loop RunDisabledStarterAsync's shutdown comment justified determinism by the background task running to completion on its own. After the disable path was made to reconcile on an interval, the loop no longer self-completes: it parks in the reconcile delay and StopAsync's cancellation is what ends it, after pass one's signal decision. Reword the comment to match; no behavior change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- .../AutoPurge/BlobPurgeJobStarterTests.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index 54be1bd0..698d3ca6 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -425,9 +425,10 @@ static EntityMetadata MetadataFor(BlobPurgeJobStatus status) "test", new TestLogger()); - // The work runs on a background task, so wait for the read before shutting down. StopAsync then waits - // for that task to finish, which makes the observation deterministic: by the time it returns the - // starter has either signalled or decided not to, rather than being timed out mid-decision. + // The work runs on a background task, so wait for the read before shutting down. The loop is perpetual, + // so it no longer ends on its own: pass one reads and then signals or decides not to before parking in + // the reconcile delay. StopAsync cancels that parked wait and awaits the task, so by the time it returns + // pass one's decision has completed and is observable, rather than being sampled mid-decision. await starter.StartAsync(CancellationToken.None); await Task.WhenAny(readCalled.Task, Task.Delay(TimeSpan.FromSeconds(30))); await starter.StopAsync(CancellationToken.None); From ce3b547e6b988f7ea22f77472863324b5ddb40c9 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 21:40:17 -0400 Subject: [PATCH 46/47] Correct stale purge-disposition docs left by the F1 quarantine change The F1 change (a814ffc2) made an unowned/unmarked blob report Quarantined instead of Deleted, but several comments and XML docs still described the old Deleted-covers-unowned behavior. Doc/comment only - no product logic or test-logic change. - src/Grpc/orchestrator_service.proto: DELETED now covers two cases (blob deleted, blob already absent), not three; the "left in place because unowned" case is QUARANTINED now. Language-neutral wire semantics only; the same hunk mirrors into microsoft/durabletask-protobuf #76. - src/Client/Core/LargePayloadPurgeDisposition.cs: same two-case fix on the C# mirror of that enum ("all three cases" -> "both cases"). - src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs: the NotStoreOwned summary no longer claims the reference "is still resolved"; it states the store-layer fact plus a neutral note that the outcome is not proof of deletion and the caller decides how to dispose of the reference. No disposition named - that mapping is the activity's decision. - src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs: same correction at the HasOwnershipMarker return site. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- src/Client/Core/LargePayloadPurgeDisposition.cs | 4 ++-- .../AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs | 5 +++-- .../AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs | 5 +++-- src/Grpc/orchestrator_service.proto | 5 ++--- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Client/Core/LargePayloadPurgeDisposition.cs b/src/Client/Core/LargePayloadPurgeDisposition.cs index bfdedca2..f48d24cf 100644 --- a/src/Client/Core/LargePayloadPurgeDisposition.cs +++ b/src/Client/Core/LargePayloadPurgeDisposition.cs @@ -15,8 +15,8 @@ public enum LargePayloadPurgeDisposition Unspecified = 0, /// - /// Terminal success. The blob was deleted, was already absent, or was deliberately left in place because - /// it is not owned by the payload store. The backend deletes the tombstone in all three cases. + /// Terminal success. The blob was deleted or was already absent. The backend deletes the tombstone in + /// both cases. /// Deleted = 1, diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index 0601751d..bb8db26e 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -288,8 +288,9 @@ public override async Task DeleteAsync(string token, Cance if (!HasOwnershipMarker(properties.Metadata)) { - // Positive evidence that the blob is customer-owned: leave it untouched. The caller still resolves - // the payload reference, because a blob this store never wrote is not this store's to delete. + // Positive evidence that the blob is customer-owned: leave it untouched. A blob this store never + // wrote is not this store's to delete, so this is not proof of deletion; the caller decides how to + // dispose of the payload reference. return PayloadDeleteOutcome.NotStoreOwned; } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs index afc95115..b7a7b6ef 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs @@ -24,8 +24,9 @@ public enum PayloadDeleteOutcome /// /// The backing object exists but does not carry the store's ownership marker, so the store did not - /// create it and left it untouched. The payload reference is still resolved, because an object the - /// store never wrote is not the store's to delete. + /// create it and left it untouched. Because an object the store never wrote is not the store's to + /// delete, this outcome is not proof that the object was deleted; the caller decides how to dispose + /// of the reference. /// NotStoreOwned, } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 080b697d..91418192 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -864,9 +864,8 @@ enum LargePayloadPurgeDisposition { LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; // Terminal success: the tombstone is resolved and the backend deletes it. Covers the blob being - // deleted, the blob already being absent, and the blob being deliberately left in place because - // the payload store does not own it. All three are terminal because none of them can be - // improved by trying again. + // deleted and the blob already being absent. Both are terminal because neither can be improved by + // trying again. LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; // The failure may self-heal, so the row stays pending and the backend sets the next attempt. From 7669cf903661dc383193a48736136a3724ff52e7 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 17 Aug 2026 22:57:18 -0400 Subject: [PATCH 47/47] Revert F1: unowned blobs report Deleted again, not Quarantined The user overruled F1. The backend treats a non-empty quarantine set as a poison/investigate signal: it emits an OTel-flagged LargePayloadPurgeQuarantineOutstanding on every GetLargePayloadTombstones poll (once per IdleDelay = 1 minute, forever), quarantine never auto-expires, and there is no operator tool to clear it. Because the managed_by=dts ownership marker is new in this PR, every pre-existing v2 blob would hit NotStoreOwned on upgrade and permanently wedge that alarm. So an unowned/unmarked blob reports Deleted (terminal success) again, as before F1. Reverts F1 only. F2 (the disable path keeps reconciling) and F3 (the two client-facing purge APIs stay removed) are unchanged. - DeleteExternalBlobActivity.cs, Logs.cs, DeleteExternalBlobActivityTests.cs: restored byte-identical to their pre-F1 (0034bb00) state. NotStoreOwned logs BlobPurgeBlobNotStoreOwned (EventId 821 restored in its 820-822 slot) and falls through to Deleted; the test asserts Deleted again. - orchestrator_service.proto, LargePayloadPurgeDisposition.cs, PayloadDeleteOutcome.cs, BlobPayloadStore.cs: the ce3b547e doc sweep only existed to describe the quarantine semantics, so it is reverted too - restored byte-identical to 9be9931b, back to the three-case DELETED wording that matches Deleted semantics and the untouched protobuf #76 copy. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6373d1ba-104b-492a-9737-456a26c8ea34 --- src/Client/Core/LargePayloadPurgeDisposition.cs | 4 ++-- .../Activities/DeleteExternalBlobActivity.cs | 17 ++++++----------- .../AzureBlobPayloads/AutoPurge/Logs.cs | 3 +++ .../PayloadStore/BlobPayloadStore.cs | 5 ++--- .../PayloadStore/PayloadDeleteOutcome.cs | 5 ++--- src/Grpc/orchestrator_service.proto | 5 +++-- .../DeleteExternalBlobActivityTests.cs | 13 ++++++------- 7 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/Client/Core/LargePayloadPurgeDisposition.cs b/src/Client/Core/LargePayloadPurgeDisposition.cs index f48d24cf..bfdedca2 100644 --- a/src/Client/Core/LargePayloadPurgeDisposition.cs +++ b/src/Client/Core/LargePayloadPurgeDisposition.cs @@ -15,8 +15,8 @@ public enum LargePayloadPurgeDisposition Unspecified = 0, /// - /// Terminal success. The blob was deleted or was already absent. The backend deletes the tombstone in - /// both cases. + /// Terminal success. The blob was deleted, was already absent, or was deliberately left in place because + /// it is not owned by the payload store. The backend deletes the tombstone in all three cases. /// Deleted = 1, diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 6be38f13..7085e1ad 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -167,21 +167,16 @@ async Task DeleteAsync(string token) using CancellationTokenSource timeout = new(this.SingleDeleteTimeout); PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, timeout.Token); - // The blob exists but this store never wrote it - it carries no ownership marker - so it is - // quarantined, not reported as deleted. The marker is new in this PR, but main has been emitting v2 - // tokens without writing it, so on upgrade every pre-existing v2 blob reaches this branch. Reporting - // Deleted here would tell the backend to hard-delete the tombstone row, orphaning that blob - // permanently AND destroying the only durable record that it exists. Quarantine keeps the row and its - // token backend-side as evidence and stops the backend polling it - the same disposition, and for the - // same reason, as the v1 and malformed-token branches: a delete that cannot be verified as this - // store's own must not be discarded as a success. + // The blob exists but this store never wrote it, so it was left untouched. That is an expected + // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it would + // fill the quarantine set with non-defects. The tombstone is still resolved, because a blob the + // store does not own is not the store's to delete. if (outcome == PayloadDeleteOutcome.NotStoreOwned) { - this.logger.BlobPurgeDeleteQuarantined("NotStoreOwned", null); - return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); + this.logger.BlobPurgeBlobNotStoreOwned(); } - // Deleted and AlreadyAbsent are terminal successes: the blob is gone, and neither can be improved by + // Deleted, AlreadyAbsent, and NotStoreOwned are all terminal successes: none can be improved by // trying again. return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Deleted); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 2499452f..66aadd15 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -43,6 +43,9 @@ static partial class Logs [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); + [LoggerMessage(EventId = 821, Level = LogLevel.Warning, Message = "An externalized payload blob does not carry this store's ownership marker, so it was left untouched; the tombstone is still resolved. This is expected for payloads written before the marker shipped, and for blobs the store never created whose token text matches the payload token grammar.")] + public static partial void BlobPurgeBlobNotStoreOwned(this ILogger logger); + [LoggerMessage(EventId = 822, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is already stopped; ignoring the stop request.")] public static partial void BlobPurgeJobAlreadyStopped(this ILogger logger, string? jobId); diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index bb8db26e..0601751d 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -288,9 +288,8 @@ public override async Task DeleteAsync(string token, Cance if (!HasOwnershipMarker(properties.Metadata)) { - // Positive evidence that the blob is customer-owned: leave it untouched. A blob this store never - // wrote is not this store's to delete, so this is not proof of deletion; the caller decides how to - // dispose of the payload reference. + // Positive evidence that the blob is customer-owned: leave it untouched. The caller still resolves + // the payload reference, because a blob this store never wrote is not this store's to delete. return PayloadDeleteOutcome.NotStoreOwned; } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs index b7a7b6ef..afc95115 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs @@ -24,9 +24,8 @@ public enum PayloadDeleteOutcome /// /// The backing object exists but does not carry the store's ownership marker, so the store did not - /// create it and left it untouched. Because an object the store never wrote is not the store's to - /// delete, this outcome is not proof that the object was deleted; the caller decides how to dispose - /// of the reference. + /// create it and left it untouched. The payload reference is still resolved, because an object the + /// store never wrote is not the store's to delete. /// NotStoreOwned, } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 91418192..080b697d 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -864,8 +864,9 @@ enum LargePayloadPurgeDisposition { LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; // Terminal success: the tombstone is resolved and the backend deletes it. Covers the blob being - // deleted and the blob already being absent. Both are terminal because neither can be improved by - // trying again. + // deleted, the blob already being absent, and the blob being deliberately left in place because + // the payload store does not own it. All three are terminal because none of them can be + // improved by trying again. LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; // The failure may self-heal, so the row stays pending and the backend sets the next attempt. diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 816caf98..893ef5fe 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -184,7 +184,7 @@ public async Task RunAsync_WhenBlobAlreadyAbsent_ReportsDeleted() } [Fact] - public async Task RunAsync_WhenBlobNotStoreOwned_Quarantines() + public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeleting() { // Arrange - the blob exists but carries no ownership marker, so the store left it untouched. Mock store = new(); @@ -196,12 +196,11 @@ public async Task RunAsync_WhenBlobNotStoreOwned_Quarantines() // Act BlobPurgeOutcome outcome = (await activity.RunAsync(null!, [V2Token]))[0]; - // Assert - quarantined, not discarded as a success. The ownership marker is new in this PR, so every v2 - // blob written before it shipped reaches this branch on upgrade; reporting Deleted would drop the - // tombstone row and orphan the blob permanently. Quarantine preserves the row and its token as evidence, - // and the logged cause is the only record of why the row reached that disposition. - outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - logger.Logs.Should().ContainSingle(l => l.Message.Contains("NotStoreOwned")); + // Assert - the tombstone is still resolved: a blob the store does not own is not the store's to delete, + // and re-serving the row forever would never make it deletable. The reported result is now identical to + // an ordinary delete, so the log is the only thing that records the difference. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("ownership marker")); } [Fact]