TST-38: Add missing concurrency stress tests for presence and webhooks

backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs

@@ -5,9 +5,13 @@
 using FluentAssertions;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.AspNetCore.SignalR.Client;
+using Microsoft.Extensions.DependencyInjection;
 using Taskdeck.Api.RateLimiting;
+using Taskdeck.Api.Realtime;
 using Taskdeck.Api.Tests.Support;
 using Taskdeck.Application.DTOs;
+using Taskdeck.Application.Interfaces;
 using Taskdeck.Domain.Entities;
 using Taskdeck.Domain.Enums;
 using Xunit;
@@ -22,7 +26,7 @@
 /// - Rate limiting under load (burst beyond limit, cross-user isolation under load)
 ///
 /// Uses Task.WhenAll with multiple HttpClient instances for HTTP-level concurrency.
-/// Uses SemaphoreSlim barriers to ensure truly simultaneous execution.
+/// Uses Barrier / SemaphoreSlim to coordinate truly simultaneous execution.
 ///
 /// NOTE: SQLite uses file-level locking for writes, so true write-contention races
 /// may serialize at the database level. These tests validate the application-layer
@@ -129,7 +133,7 @@
             async () =>
             {
                 var r = await client.GetAsync($"/api/capture/items/{capture.Id}");
                 return await r.Content.ReadFromJsonAsync<CaptureItemDto>();
             },
             item => item?.Status is CaptureStatus.ProposalCreated or CaptureStatus.Triaging,
             "capture triage processing",
@@ -473,7 +477,7 @@
             async () =>
             {
                 var r = await client.GetAsync($"/api/capture/items/{capture.Id}");
                 return await r.Content.ReadFromJsonAsync<CaptureItemDto>();
             },
             item => item?.Status == CaptureStatus.ProposalCreated,
             "proposal creation from capture triage",
@@ -541,7 +545,7 @@
             async () =>
             {
                 var r = await client.GetAsync($"/api/capture/items/{capture.Id}");
                 return await r.Content.ReadFromJsonAsync<CaptureItemDto>();
             },
             item => item?.Status == CaptureStatus.ProposalCreated,
             "proposal creation for approve vs reject race",
@@ -627,7 +631,7 @@
             async () =>
             {
                 var r = await client.GetAsync($"/api/capture/items/{capture.Id}");
                 return await r.Content.ReadFromJsonAsync<CaptureItemDto>();
             },
             item => item?.Status == CaptureStatus.ProposalCreated,
             "proposal creation for double execute race",
@@ -821,4 +825,371 @@
         userBoards.Should().HaveCount(userCount,
             "all users should have created their boards successfully");
     }
+
+    // ── SignalR Presence Concurrency ────────────────────────────────────────
+
+    /// <summary>
+    /// Polls the observer's event collector until a snapshot with the expected
+    /// member count appears, or the timeout elapses. This avoids flakiness
+    /// from checking only the last event, which may not reflect the settled state.
+    /// </summary>
+    private static async Task<BoardPresenceSnapshot> WaitForPresenceCountAsync(
+        EventCollector<BoardPresenceSnapshot> events,
+        int expectedMemberCount,
+        TimeSpan? timeout = null)
+    {
+        var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(10);
+        var deadline = DateTimeOffset.UtcNow + effectiveTimeout;
+        while (DateTimeOffset.UtcNow < deadline)
+        {
+            var snapshot = events.ToList().LastOrDefault();
+            if (snapshot is not null && snapshot.Members.Count == expectedMemberCount)
+            {
+                return snapshot;
+            }
+
+            await Task.Delay(50);
+        }
+
+        var last = events.ToList().LastOrDefault();
+        var actualCount = last?.Members.Count ?? 0;
+        throw new TimeoutException(
+            $"Expected presence snapshot with {expectedMemberCount} members but last snapshot had {actualCount} within {effectiveTimeout.TotalSeconds}s.");
+    }
+
+    /// <summary>
+    /// Scenario 14: Rapid join/leave stress.
+    /// Multiple connections rapidly join and leave a board.
+    /// After all connections settle, the final presence snapshot should be
+    /// eventually consistent (only connections that remain joined are present).
+    /// </summary>
+    [Fact]
+    public async Task Presence_RapidJoinLeave_EventuallyConsistent()
+    {
+        const int connectionCount = 5;
+
+        using var ownerClient = _factory.CreateClient();
+        var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-presence-rapid");
+        var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-rapid-board");
+
+        // Create users and grant access — reuse a single HttpClient for setup
+        using var setupClient = _factory.CreateClient();
+        var users = new List<TestUserContext>();
+        for (var i = 0; i < connectionCount; i++)
+        {
+            var u = await ApiTestHarness.AuthenticateAsync(setupClient, $"race-rapid-{i}");
+            var grant = await ownerClient.PostAsJsonAsync(
+                $"/api/boards/{board.Id}/access",
+                new GrantAccessDto(board.Id, u.UserId, UserRole.Editor));
+            grant.StatusCode.Should().Be(HttpStatusCode.OK);
+            users.Add(u);
+        }
+
+        // Owner observes presence events
+        var observerEvents = new EventCollector<BoardPresenceSnapshot>();
+        await using var observer = SignalRTestHelper.CreateBoardsHubConnection(_factory, owner.Token);
+        observer.On<BoardPresenceSnapshot>("boardPresence", snapshot => observerEvents.Add(snapshot));
+        await observer.StartAsync();
+        await observer.InvokeAsync("JoinBoard", board.Id);
+        await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1);
+        observerEvents.Clear();
+
+        // All users join simultaneously via Barrier for true synchronization.
+        // Unlike SemaphoreSlim, Barrier ensures all participants reach the
+        // barrier point before any of them proceed past it.
+        var connections = new List<HubConnection>();
+        try
+        {
+            using var joinBarrier = new Barrier(connectionCount + 1);
+            var joinTasks = users.Select(async user =>
+            {
+                var conn = SignalRTestHelper.CreateBoardsHubConnection(_factory, user.Token);
+                conn.On<BoardPresenceSnapshot>("boardPresence", _ => { });
+                await conn.StartAsync();
+                lock (connections) { connections.Add(conn); }
+                joinBarrier.SignalAndWait(TimeSpan.FromSeconds(10));
+                await conn.InvokeAsync("JoinBoard", board.Id);
+            }).ToArray();
+
+            joinBarrier.SignalAndWait(TimeSpan.FromSeconds(10));
+            await Task.WhenAll(joinTasks);
+
+            // Poll until the observer snapshot settles at the expected member count
+            // (all joined users plus the owner). This avoids flakiness from
+            // intermediate snapshots that don't yet reflect all joins.
+            var afterJoin = await WaitForPresenceCountAsync(
+                observerEvents, connectionCount + 1, TimeSpan.FromSeconds(10));
+            afterJoin.Members.Should().HaveCount(connectionCount + 1,
+                "all joined users plus the observer owner should be present");
+
+            // Now have the first half leave rapidly
+            observerEvents.Clear();
+            var leavingCount = connectionCount / 2;
+            using var leaveBarrier = new Barrier(leavingCount + 1);
+            var leaveTasks = connections.Take(leavingCount).Select(async conn =>
+            {
+                leaveBarrier.SignalAndWait(TimeSpan.FromSeconds(10));
+                await conn.InvokeAsync("LeaveBoard", board.Id);
+            }).ToArray();
+
+            leaveBarrier.SignalAndWait(TimeSpan.FromSeconds(10));
+            await Task.WhenAll(leaveTasks);
+
+            // Poll until the snapshot settles at the expected remaining count
+            var remaining = connectionCount - leavingCount;
+            var afterLeave = await WaitForPresenceCountAsync(
+                observerEvents, remaining + 1, TimeSpan.FromSeconds(10));
+            afterLeave.Members.Should().HaveCount(remaining + 1,
+                $"after {leavingCount} leaves, {remaining} users + owner should remain");
+        }
+        finally
+        {
+            foreach (var conn in connections)
+            {
+                await conn.DisposeAsync();
+            }
+        }
+    }
+
+    /// <summary>
+    /// Scenario 15: Disconnect during edit clears editing state.
+    /// A user sets an editing card, then their connection drops abruptly.
+    /// The presence snapshot should no longer include the editing state.
+    /// </summary>
+    [Fact]
+    public async Task Presence_DisconnectDuringEdit_ClearsEditingState()
+    {
+        using var ownerClient = _factory.CreateClient();
+        var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-disc-edit");
+        var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-disc-edit-board");
+
+        // Create a column and card
+        var colResp = await ownerClient.PostAsJsonAsync(
+            $"/api/boards/{board.Id}/columns",
+            new CreateColumnDto(board.Id, "Backlog", null, null));
+        colResp.StatusCode.Should().Be(HttpStatusCode.Created);
+        var col = await colResp.Content.ReadFromJsonAsync<ColumnDto>();
+
+        var cardResp = await ownerClient.PostAsJsonAsync(
+            $"/api/boards/{board.Id}/cards",
+            new CreateCardDto(board.Id, col!.Id, "Edit-then-disconnect card", null, null, null));
+        cardResp.StatusCode.Should().Be(HttpStatusCode.Created);
+        var card = await cardResp.Content.ReadFromJsonAsync<CardDto>();
+
+        // Second user who will disconnect
+        using var editorClient = _factory.CreateClient();
+        var editor = await ApiTestHarness.AuthenticateAsync(editorClient, "race-disc-editor");
+        var grant = await ownerClient.PostAsJsonAsync(
+            $"/api/boards/{board.Id}/access",
+            new GrantAccessDto(board.Id, editor.UserId, UserRole.Editor));
+        grant.StatusCode.Should().Be(HttpStatusCode.OK);
+
+        // Owner joins and observes
+        var observerEvents = new EventCollector<BoardPresenceSnapshot>();
+        await using var observer = SignalRTestHelper.CreateBoardsHubConnection(_factory, owner.Token);
+        observer.On<BoardPresenceSnapshot>("boardPresence", snapshot => observerEvents.Add(snapshot));
+        await observer.StartAsync();
+        await observer.InvokeAsync("JoinBoard", board.Id);
+        await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1);
+
+        // Editor joins and starts editing.
+        // We use 'await using' so the connection is always disposed (even if an
+        // assertion throws). The abrupt disconnect is simulated by disposing
+        // without calling LeaveBoard or SetEditingCard(null) first.
+        await using var editorConn = SignalRTestHelper.CreateBoardsHubConnection(_factory, editor.Token);
+        editorConn.On<BoardPresenceSnapshot>("boardPresence", _ => { });
+        await editorConn.StartAsync();
+        await editorConn.InvokeAsync("JoinBoard", board.Id);
+        await SignalRTestHelper.WaitForEventsAsync(observerEvents, 2); // join event
+
+        observerEvents.Clear();
+        await editorConn.InvokeAsync("SetEditingCard", board.Id, card!.Id);
+
+        // Wait for editing presence update
+        var editingEvents = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1);
+        var editorMember = editingEvents.Last().Members
+            .FirstOrDefault(m => m.UserId == editor.UserId);
+        editorMember.Should().NotBeNull("editor should be visible in presence");
+        editorMember!.EditingCardId.Should().Be(card.Id,
+            "editor should show as editing the card");
+
+        // Abrupt disconnect (no LeaveBoard, no SetEditingCard(null))
+        observerEvents.Clear();
+        await editorConn.DisposeAsync();
+
+        // Owner should receive a snapshot without the editor
+        var afterDisconnect = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1,
+            TimeSpan.FromSeconds(5));
+        afterDisconnect.Last().Members.Should().NotContain(m => m.UserId == editor.UserId,
+            "disconnected editor should be removed from presence");
+        afterDisconnect.Last().Members.Should().ContainSingle(m => m.UserId == owner.UserId,
+            "only the owner should remain in presence after editor disconnects");
+    }
+
+    // ── Concurrent Webhook Delivery Creation ────────────────────────────────
+
+    /// <summary>
+    /// Scenario 16: Concurrent board mutations should each create webhook deliveries.
+    /// Multiple card operations fire concurrently on a board with an active
+    /// webhook subscription. Each mutation should produce its own delivery
+    /// record without duplicates or lost events.
+    /// </summary>
+    [Fact]
+    public async Task WebhookDelivery_ConcurrentBoardMutations_EachCreatesDeliveryRecord()
+    {
+        const int mutationCount = 5;
+
+        using var client = _factory.CreateClient();
+        await ApiTestHarness.AuthenticateAsync(client, "race-webhook-delivery");
+        var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-board");
+
+        var colResp = await client.PostAsJsonAsync(
+            $"/api/boards/{board.Id}/columns",
+            new CreateColumnDto(board.Id, "Backlog", null, null));
+        colResp.StatusCode.Should().Be(HttpStatusCode.Created);
+        var col = await colResp.Content.ReadFromJsonAsync<ColumnDto>();
+
+        // Create a webhook subscription on this board.
+        // NOTE: the endpoint URL is external-looking; delivery will fail at send time
+        // (no real server) but the delivery RECORD should be created.
+        var webhookResp = await client.PostAsJsonAsync(
+            $"/api/boards/{board.Id}/webhooks",
+            new CreateOutboundWebhookSubscriptionDto(
+                "https://example.com/webhook-receiver",
+                new List<string> { "card.*" }));
+        webhookResp.StatusCode.Should().Be(HttpStatusCode.Created);
+        var webhookSub = await webhookResp.Content.ReadFromJsonAsync<OutboundWebhookSubscriptionSecretDto>();
+        webhookSub.Should().NotBeNull("webhook subscription should have been created");
+
+        // Create multiple cards concurrently — each should trigger a webhook delivery.
+        // Use Barrier for true synchronization instead of SemaphoreSlim.
+        using var barrier = new Barrier(mutationCount + 1);
+        var statusCodes = new ConcurrentBag<HttpStatusCode>();
+
+        var mutationTasks = Enumerable.Range(0, mutationCount).Select(async i =>
+        {
+            using var raceClient = _factory.CreateClient();
+            raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization;
+            barrier.SignalAndWait(TimeSpan.FromSeconds(10));
+            var resp = await raceClient.PostAsJsonAsync(
+                $"/api/boards/{board.Id}/cards",
+                new CreateCardDto(board.Id, col!.Id, $"Webhook card {i}", null, null, null));
+            statusCodes.Add(resp.StatusCode);
+        }).ToArray();
+
+        barrier.SignalAndWait(TimeSpan.FromSeconds(10));
+        await Task.WhenAll(mutationTasks);
+
+        // All card creations should succeed
+        statusCodes.Should().AllSatisfy(s =>
+            s.Should().Be(HttpStatusCode.Created),
+            "all concurrent card creations should succeed");
+
+        // Verify all cards were created (no duplicates, no losses)
+        var cardsResp = await client.GetAsync($"/api/boards/{board.Id}/cards");
+        cardsResp.StatusCode.Should().Be(HttpStatusCode.OK);
+        var cards = await cardsResp.Content.ReadFromJsonAsync<List<CardDto>>();
+        var webhookCards = cards!.Where(c => c.Title.StartsWith("Webhook card ")).ToList();
+        webhookCards.Should().HaveCount(mutationCount,
+            "each concurrent mutation should create exactly one card (no duplicates or losses)");
+
+        // Verify all card titles are unique (no duplicate processing)
+        webhookCards.Select(c => c.Title).Distinct().Should().HaveCount(mutationCount,
+            "each card title should be unique, proving no duplicate processing");
+
+        // Verify that webhook delivery records were actually created in the database.
+        // Poll with a short timeout because the notifier enqueues deliveries
+        // asynchronously after the HTTP response returns.
+        using var scope = _factory.Services.CreateScope();
+        var deliveryRepo = scope.ServiceProvider.GetRequiredService<IOutboundWebhookDeliveryRepository>();
+
+        var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10);
+        IReadOnlyList<OutboundWebhookDelivery> deliveries = [];
+        while (DateTimeOffset.UtcNow < deadline)
+        {
+            deliveries = await deliveryRepo.GetBySubscriptionAsync(
+                webhookSub!.Subscription.Id, limit: mutationCount + 5);
+            if (deliveries.Count >= mutationCount)
+            {
+                break;
+            }
+
+            await Task.Delay(100);
+        }
+
+        deliveries.Should().HaveCount(mutationCount,
+            $"each of the {mutationCount} card mutations should create exactly one webhook delivery record");
+        deliveries.Select(d => d.Id).Distinct().Should().HaveCount(deliveries.Count,
+            "each delivery record should have a unique ID");
+    }
+
+    /// <summary>
+    /// Scenario 17: Concurrent webhook subscription creation on the same board.
+    /// Multiple webhook subscriptions created simultaneously should all succeed
+    /// with distinct IDs and secrets.
+    /// </summary>
+    [Fact]
+    public async Task WebhookSubscription_ConcurrentCreation_AllSucceedWithDistinctIds()
+    {
+        const int subscriptionCount = 3;
+
+        using var client = _factory.CreateClient();
+        await ApiTestHarness.AuthenticateAsync(client, "race-webhook-sub");
+        var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-sub-board");
+
+        // Use Barrier for true synchronization instead of SemaphoreSlim
+        using var barrier = new Barrier(subscriptionCount + 1);
+        var results = new ConcurrentBag<(HttpStatusCode Status, OutboundWebhookSubscriptionSecretDto? Sub)>();
+
+        var tasks = Enumerable.Range(0, subscriptionCount).Select(async i =>
+        {
+            using var raceClient = _factory.CreateClient();
+            raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization;
+            barrier.SignalAndWait(TimeSpan.FromSeconds(10));
+            var resp = await raceClient.PostAsJsonAsync(
+                $"/api/boards/{board.Id}/webhooks",
+                new CreateOutboundWebhookSubscriptionDto(
+                    $"https://example.com/webhook-{i}",
+                    new List<string> { "card.*" }));
+            var sub = resp.StatusCode == HttpStatusCode.Created
+                ? await resp.Content.ReadFromJsonAsync<OutboundWebhookSubscriptionSecretDto>()
+                : null;
+            results.Add((resp.StatusCode, sub));
+        }).ToArray();
+
+        barrier.SignalAndWait(TimeSpan.FromSeconds(10));
+        await Task.WhenAll(tasks);
+
+        // All should succeed
+        results.Select(r => r.Status).Should().AllSatisfy(s =>
+            s.Should().Be(HttpStatusCode.Created),
+            "all concurrent webhook subscription creations should succeed");
+
+        // IDs should be distinct
+        var ids = results.Where(r => r.Sub != null).Select(r => r.Sub!.Subscription.Id).ToList();
+        ids.Distinct().Should().HaveCount(subscriptionCount,
+            "each subscription should have a unique ID");
+
+        // Signing secrets should be distinct
+        var secrets = results.Where(r => r.Sub != null).Select(r => r.Sub!.SigningSecret).ToList();
+        secrets.Distinct().Should().HaveCount(subscriptionCount,
+            "each subscription should have a unique signing secret");
+
+        // Verify via list endpoint: parse response and verify distinct IDs and count
+        var listResp = await client.GetAsync($"/api/boards/{board.Id}/webhooks");
+        listResp.StatusCode.Should().Be(HttpStatusCode.OK);
+        var listedSubs = await listResp.Content.ReadFromJsonAsync<List<OutboundWebhookSubscriptionDto>>()
+            ?? throw new InvalidOperationException("list endpoint returned null subscription data");
+        listedSubs.Should().HaveCountGreaterThanOrEqualTo(subscriptionCount,
+            $"list endpoint should return at least {subscriptionCount} subscriptions");
+        listedSubs.Select(s => s.Id).Distinct().Should().HaveCountGreaterThanOrEqualTo(subscriptionCount,
+            "listed subscriptions should have distinct IDs");
+
+        // Cross-check: all IDs from creation should appear in the list
+        foreach (var createdId in ids)
+        {
+            listedSubs.Should().Contain(s => s.Id == createdId,
+                $"subscription {createdId} created concurrently should appear in the list endpoint");
+        }
+    }
 }