Reduce sync work: faster snapshot projection + benchmarks

Directory.Packages.props

@@ -16,7 +16,7 @@
     <PackageVersion Include="Nito.AsyncEx.Coordination" Version="5.1.2" />
     <PackageVersion Include="System.IO.Hashing" Version="10.0.7" />
     <PackageVersion Include="System.Linq.Async" Version="7.0.1" />
-    <PackageVersion Include="BenchmarkDotNet" Version="0.15.8" />
+    <PackageVersion Include="BenchmarkDotNet" Version="0.16.0-preview.1" />
     <PackageVersion Include="FluentAssertions" Version="8.9.0" />
     <PackageVersion Include="GitHubActionsTestLogger" Version="3.0.5" />
     <PackageVersion Include="JetBrains.Profiler.SelfApi" Version="2.5.18" />
@@ -31,4 +31,4 @@
     <PackageVersion Include="linq2db.Extensions" Version="6.2.1" />
     <PackageVersion Include="EFCore.ComplexIndexes" Version="3.1.5" />
   </ItemGroup>
-</Project>
+</Project>

harmony.slnx

@@ -7,6 +7,7 @@
   <Project Path="src/SIL.Harmony.Core/SIL.Harmony.Core.csproj" />
   <Project Path="src/SIL.Harmony.Linq2db/SIL.Harmony.Linq2db.csproj" />
   <Project Path="src/SIL.Harmony.Sample/SIL.Harmony.Sample.csproj" />
+  <Project Path="src/SIL.Harmony.Benchmarks/SIL.Harmony.Benchmarks.csproj" />
   <Project Path="src/SIL.Harmony.Tests/SIL.Harmony.Tests.csproj" />
   <Project Path="src/SIL.Harmony/SIL.Harmony.csproj" />
   <Project Path="src/Ycs/Ycs.csproj" />

src/SIL.Harmony.Benchmarks/AddSnapshotsBenchmarks.cs

@@ -0,0 +1,157 @@
+using System.Diagnostics.CodeAnalysis;
+using BenchmarkDotNet.Attributes;
+using Microsoft.EntityFrameworkCore;
+using SIL.Harmony.Db;
+using SIL.Harmony.Tests;
+
+namespace SIL.Harmony.Benchmarks;
+
+public enum AddSnapshotsWorkload
+{
+    /// <summary>Create many distinct entities (root snapshots, all inserts, no FindAsync hits).</summary>
+    CreateNew,
+
+    /// <summary>Word + Definition per commit — two projected table types.</summary>
+    MultiTypeCreate,
+
+    /// <summary>Word + Tag + WordTag per commit — three types plus a reference graph.</summary>
+    ReferencedCreate,
+
+    /// <summary>Seed created words, then update each once — updates project onto existing rows (FindAsync per snapshot).</summary>
+    UpdateExisting,
+
+    /// <summary>Modify one entity many times — large snapshot count (many intermediates), dedup, few projected rows.</summary>
+    ModifySameEntity,
+
+    /// <summary>Create then delete each entity — delete group + insert group, two SaveChanges.</summary>
+    CreateThenDelete,
+
+    /// <summary>Create, delete, then apply-on-deleted — mixed EntityIsDeleted projection skips.</summary>
+    CreateDeleteModify,
+}
+
+// Isolates CrdtRepository.AddSnapshots from the rest of the sync pipeline.
+// Expensive DB seeding happens once in GlobalSetup; each iteration gets a clean copy via ForkDatabase() and
+// recomputes the snapshot batch so no EF-tracked state leaks across iterations.
+// disable warning about waiting for sync code, benchmarkdotnet does not support async code, and it doesn't deadlock when waiting.
+[SuppressMessage("Usage", "VSTHRD002:Avoid problematic synchronous waits")]
+public class AddSnapshotsBenchmarks
+{
+    private DataModelTestBase _template = null!;
+    private HashSet<Guid> _measuredCommitIds = null!;
+
+    private DataModelTestBase _local = null!;
+    private CrdtRepository _repository = null!;
+    private ObjectSnapshot[] _snapshotsToAdd = null!;
+
+    [Params(1000, 10_000)]
+    public int ChangeCount { get; set; }
+
+    [ParamsAllValues]
+    public AddSnapshotsWorkload Workload { get; set; }
+
+    [GlobalSetup]
+    public void GlobalSetup()
+    {
+        _template = new DataModelTestBase(alwaysValidate: false, performanceTest: true);
+        var clientId = Guid.NewGuid();
+
+        List<Commit> seed;
+        List<Commit> measured;
+        switch (Workload)
+        {
+            case AddSnapshotsWorkload.CreateNew:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildCreateWords(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.MultiTypeCreate:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildWordsWithDefinitions(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.ReferencedCreate:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildWordsWithTags(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.UpdateExisting:
+                (seed, measured) = BenchmarkWorkloadBuilders.BuildUpdateExisting(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.ModifySameEntity:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildModifySameWord(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.CreateThenDelete:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildCreateThenDelete(_template, clientId, ChangeCount);
+                break;
+            case AddSnapshotsWorkload.CreateDeleteModify:
+                seed = [];
+                measured = BenchmarkWorkloadBuilders.BuildCreateDeleteModify(_template, clientId, ChangeCount);
+                break;
+            default:
+                throw new ArgumentOutOfRangeException();
+        }
+
+        // Seed commits go through the full pipeline so their snapshots and projected rows already exist.
+        if (seed.Count > 0)
+            ((ISyncable)_template.DataModel).AddRangeFromSync(seed).Wait();
+
+        // The measured commits are present in the database but their snapshots are NOT yet persisted; that's the
+        // work AddSnapshots performs. Adding only the commits mirrors the state right before UpdateSnapshots runs.
+        var repository = _template.CreateRepository();
+        repository.AddCommits(measured).GetAwaiter().GetResult();
+        _measuredCommitIds = measured.Select(c => c.Id).ToHashSet();
+    }
+
+    [IterationSetup]
+    public void IterationSetup()
+    {
+        _local = _template.ForkDatabase(alwaysValidate: false);
+        _repository = _local.CreateRepository();
+
+        // Load the measured commits fresh from the fork (tracked) so AddSnapshots resolves their Commit navigation
+        // from the change tracker instead of trying to re-insert them.
+        var measuredCommits = _local.DbContext.Commits
+            .Include(c => c.ChangeEntities)
+            .Where(c => EF.Parameter(_measuredCommitIds).Contains(c.Id))
+            .ToArray()
+            .ToSortedSet();
+
+        // Prepopulate the snapshot lookup the same way DataModel.UpdateSnapshots does: existing snapshots (untracked)
+        // plus null for entities without one, so SnapshotWorker doesn't issue a per-entity query while computing.
+        var entityIds = measuredCommits
+            .SelectMany(c => c.ChangeEntities.Select(ce => ce.EntityId))
+            .ToHashSet();
+        var snapshotLookup = _repository.CurrentSnapshots()
+            .Include(s => s.Commit)
+            .Where(s => EF.Parameter(entityIds).Contains(s.EntityId))
+            .ToDictionary(s => s.EntityId, s => (ObjectSnapshot?)s);
+        foreach (var entityId in entityIds)
+            snapshotLookup.TryAdd(entityId, null);
+
+        var worker = new SnapshotWorker(snapshotLookup, _repository, _local.CrdtConfig);
+        _snapshotsToAdd = worker.ComputeSnapshotsToPersist(measuredCommits).GetAwaiter().GetResult().ToArray();
+    }
+
+    [Benchmark]
+    public void AddSnapshots()
+    {
+        _repository.AddSnapshots(_snapshotsToAdd).Wait();
+    }
+
+    [IterationCleanup]
+    public void IterationCleanup()
+    {
+        _repository.DisposeAsync().AsTask().Wait();
+        _local.DisposeAsync().AsTask().Wait();
+        _repository = null!;
+        _local = null!;
+        _snapshotsToAdd = null!;
+    }
+
+    [GlobalCleanup]
+    public void GlobalCleanup()
+    {
+        _template.DisposeAsync().AsTask().Wait();
+        _template = null!;
+    }
+}

src/SIL.Harmony.Benchmarks/BenchmarkWorkloadBuilders.cs

@@ -0,0 +1,168 @@
+using SIL.Harmony.Changes;
+using SIL.Harmony.Tests;
+
+namespace SIL.Harmony.Benchmarks;
+
+/// <summary>
+/// Shared commit-building helpers for the benchmark suites so <see cref="DataModelSyncBenchmarks"/> (full sync
+/// pipeline) and <see cref="AddSnapshotsBenchmarks"/> (isolated snapshot persist) exercise the same domain scenarios.
+/// Builders are pure: they only use the source <see cref="DataModelTestBase"/> for its change factories and
+/// <see cref="DataModelTestBase.NextDate"/> counter and never touch the database.
+/// </summary>
+public static class BenchmarkWorkloadBuilders
+{
+    public static Commit NewCommit(Guid clientId, DateTimeOffset dateTime, params IChange[] changes)
+    {
+        var commit = new Commit(Guid.NewGuid())
+        {
+            ClientId = clientId,
+            HybridDateTime = new HybridDateTime(dateTime, 0)
+        };
+        for (var i = 0; i < changes.Length; i++)
+        {
+            commit.ChangeEntities.Add(DataModel.ToChangeEntity(changes[i], i, commit.Id));
+        }
+        return commit;
+    }
+
+    /// <summary>Create many distinct words (one SetWord per commit).</summary>
+    public static List<Commit> BuildCreateWords(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(Guid.NewGuid(), $"entity {i}")));
+        }
+        return commits;
+    }
+
+    /// <summary>Create words each with a new definition in the same commit.</summary>
+    public static List<Commit> BuildWordsWithDefinitions(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}"),
+                src.NewDefinition(wordId, $"definition {i}", "noun")));
+        }
+        return commits;
+    }
+
+    /// <summary>Create words each with a tag and WordTag link in the same commit.</summary>
+    public static List<Commit> BuildWordsWithTags(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            var tagId = Guid.NewGuid();
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}"),
+                src.SetTag(tagId, $"tag {i}"),
+                src.TagWord(wordId, tagId)));
+        }
+        return commits;
+    }
+
+    /// <summary>Create one word, then modify that same word repeatedly.</summary>
+    public static List<Commit> BuildModifySameWord(DataModelTestBase src, Guid clientId, int count)
+    {
+        var wordId = Guid.NewGuid();
+        var commits = new List<Commit>(count);
+        commits.Add(NewCommit(clientId, src.NextDate(),
+            src.SetWord(wordId, "entity 0")));
+        for (var i = 1; i < count; i++)
+        {
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}")));
+        }
+        return commits;
+    }
+
+    /// <summary>Create words then delete them.</summary>
+    public static List<Commit> BuildCreateThenDelete(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count * 2);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}")));
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.DeleteWord(wordId)));
+        }
+        return commits;
+    }
+
+    /// <summary>Create, delete, then modify (apply-after-delete) for each word.</summary>
+    public static List<Commit> BuildCreateDeleteModify(DataModelTestBase src, Guid clientId, int count)
+    {
+        var commits = new List<Commit>(count * 3);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordId, $"entity {i}")));
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.DeleteWord(wordId)));
+            // SetWordNote supports apply-on-existing (including deleted) without undeleting
+            commits.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWordNote(wordId, $"note {i}")));
+        }
+        return commits;
+    }
+
+    /// <summary>
+    /// Local already has create + late modify; sync inserts a mid-history modify for each word
+    /// (forces stale snapshot deletion / rebuild). Returns the seed commits and the mid-history commits to sync.
+    /// </summary>
+    public static (List<Commit> seed, List<Commit> toSync) BuildOutOfOrderInsert(DataModelTestBase src, Guid clientId, int count)
+    {
+        var seed = new List<Commit>(count * 2);
+        var toSync = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            var wordId = Guid.NewGuid();
+            var createTime = src.NextDate();
+            var midTime = src.NextDate();
+            var lateTime = src.NextDate();
+
+            seed.Add(NewCommit(clientId, createTime,
+                src.SetWord(wordId, $"entity {i}")));
+            // Mid-history change is what gets synced after local already has create + late
+            toSync.Add(NewCommit(clientId, midTime,
+                src.SetWordNote(wordId, $"note {i}")));
+            seed.Add(NewCommit(clientId, lateTime,
+                src.SetWord(wordId, $"entity {i} late")));
+        }
+
+        return (seed, toSync);
+    }
+
+    /// <summary>
+    /// Seed a set of created words, then modify each one exactly once. The updates are the measured batch;
+    /// their snapshots must update the already-projected rows (FindAsync per snapshot in the slow path).
+    /// </summary>
+    public static (List<Commit> seed, List<Commit> measured) BuildUpdateExisting(DataModelTestBase src, Guid clientId, int count)
+    {
+        var wordIds = new Guid[count];
+        var seed = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            wordIds[i] = Guid.NewGuid();
+            seed.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordIds[i], $"entity {i}")));
+        }
+
+        var measured = new List<Commit>(count);
+        for (var i = 0; i < count; i++)
+        {
+            measured.Add(NewCommit(clientId, src.NextDate(),
+                src.SetWord(wordIds[i], $"entity {i} updated")));
+        }
+        return (seed, measured);
+    }
+}