diff --git a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs index 4c1ccf92..9cc84a7a 100644 --- a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs @@ -5,9 +5,13 @@ using FluentAssertions; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.Extensions.DependencyInjection; using Taskdeck.Api.RateLimiting; +using Taskdeck.Api.Realtime; using Taskdeck.Api.Tests.Support; using Taskdeck.Application.DTOs; +using Taskdeck.Application.Interfaces; using Taskdeck.Domain.Entities; using Taskdeck.Domain.Enums; using Xunit; @@ -22,7 +26,7 @@ namespace Taskdeck.Api.Tests; /// - Rate limiting under load (burst beyond limit, cross-user isolation under load) /// /// Uses Task.WhenAll with multiple HttpClient instances for HTTP-level concurrency. -/// Uses SemaphoreSlim barriers to ensure truly simultaneous execution. +/// Uses Barrier / SemaphoreSlim to coordinate truly simultaneous execution. /// /// NOTE: SQLite uses file-level locking for writes, so true write-contention races /// may serialize at the database level. These tests validate the application-layer @@ -821,4 +825,371 @@ public async Task BoardCreation_ConcurrentMultiUser_NoCrossContamination() userBoards.Should().HaveCount(userCount, "all users should have created their boards successfully"); } + + // ── SignalR Presence Concurrency ──────────────────────────────────────── + + /// + /// Polls the observer's event collector until a snapshot with the expected + /// member count appears, or the timeout elapses. This avoids flakiness + /// from checking only the last event, which may not reflect the settled state. + /// + private static async Task WaitForPresenceCountAsync( + EventCollector events, + int expectedMemberCount, + TimeSpan? timeout = null) + { + var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(10); + var deadline = DateTimeOffset.UtcNow + effectiveTimeout; + while (DateTimeOffset.UtcNow < deadline) + { + var snapshot = events.ToList().LastOrDefault(); + if (snapshot is not null && snapshot.Members.Count == expectedMemberCount) + { + return snapshot; + } + + await Task.Delay(50); + } + + var last = events.ToList().LastOrDefault(); + var actualCount = last?.Members.Count ?? 0; + throw new TimeoutException( + $"Expected presence snapshot with {expectedMemberCount} members but last snapshot had {actualCount} within {effectiveTimeout.TotalSeconds}s."); + } + + /// + /// Scenario 14: Rapid join/leave stress. + /// Multiple connections rapidly join and leave a board. + /// After all connections settle, the final presence snapshot should be + /// eventually consistent (only connections that remain joined are present). + /// + [Fact] + public async Task Presence_RapidJoinLeave_EventuallyConsistent() + { + const int connectionCount = 5; + + using var ownerClient = _factory.CreateClient(); + var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-presence-rapid"); + var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-rapid-board"); + + // Create users and grant access — reuse a single HttpClient for setup + using var setupClient = _factory.CreateClient(); + var users = new List(); + for (var i = 0; i < connectionCount; i++) + { + var u = await ApiTestHarness.AuthenticateAsync(setupClient, $"race-rapid-{i}"); + var grant = await ownerClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/access", + new GrantAccessDto(board.Id, u.UserId, UserRole.Editor)); + grant.StatusCode.Should().Be(HttpStatusCode.OK); + users.Add(u); + } + + // Owner observes presence events + var observerEvents = new EventCollector(); + await using var observer = SignalRTestHelper.CreateBoardsHubConnection(_factory, owner.Token); + observer.On("boardPresence", snapshot => observerEvents.Add(snapshot)); + await observer.StartAsync(); + await observer.InvokeAsync("JoinBoard", board.Id); + await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); + observerEvents.Clear(); + + // All users join simultaneously via Barrier for true synchronization. + // Unlike SemaphoreSlim, Barrier ensures all participants reach the + // barrier point before any of them proceed past it. + var connections = new List(); + try + { + using var joinBarrier = new Barrier(connectionCount + 1); + var joinTasks = users.Select(async user => + { + var conn = SignalRTestHelper.CreateBoardsHubConnection(_factory, user.Token); + conn.On("boardPresence", _ => { }); + await conn.StartAsync(); + lock (connections) { connections.Add(conn); } + joinBarrier.SignalAndWait(TimeSpan.FromSeconds(10)); + await conn.InvokeAsync("JoinBoard", board.Id); + }).ToArray(); + + joinBarrier.SignalAndWait(TimeSpan.FromSeconds(10)); + await Task.WhenAll(joinTasks); + + // Poll until the observer snapshot settles at the expected member count + // (all joined users plus the owner). This avoids flakiness from + // intermediate snapshots that don't yet reflect all joins. + var afterJoin = await WaitForPresenceCountAsync( + observerEvents, connectionCount + 1, TimeSpan.FromSeconds(10)); + afterJoin.Members.Should().HaveCount(connectionCount + 1, + "all joined users plus the observer owner should be present"); + + // Now have the first half leave rapidly + observerEvents.Clear(); + var leavingCount = connectionCount / 2; + using var leaveBarrier = new Barrier(leavingCount + 1); + var leaveTasks = connections.Take(leavingCount).Select(async conn => + { + leaveBarrier.SignalAndWait(TimeSpan.FromSeconds(10)); + await conn.InvokeAsync("LeaveBoard", board.Id); + }).ToArray(); + + leaveBarrier.SignalAndWait(TimeSpan.FromSeconds(10)); + await Task.WhenAll(leaveTasks); + + // Poll until the snapshot settles at the expected remaining count + var remaining = connectionCount - leavingCount; + var afterLeave = await WaitForPresenceCountAsync( + observerEvents, remaining + 1, TimeSpan.FromSeconds(10)); + afterLeave.Members.Should().HaveCount(remaining + 1, + $"after {leavingCount} leaves, {remaining} users + owner should remain"); + } + finally + { + foreach (var conn in connections) + { + await conn.DisposeAsync(); + } + } + } + + /// + /// Scenario 15: Disconnect during edit clears editing state. + /// A user sets an editing card, then their connection drops abruptly. + /// The presence snapshot should no longer include the editing state. + /// + [Fact] + public async Task Presence_DisconnectDuringEdit_ClearsEditingState() + { + using var ownerClient = _factory.CreateClient(); + var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-disc-edit"); + var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-disc-edit-board"); + + // Create a column and card + var colResp = await ownerClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/columns", + new CreateColumnDto(board.Id, "Backlog", null, null)); + colResp.StatusCode.Should().Be(HttpStatusCode.Created); + var col = await colResp.Content.ReadFromJsonAsync(); + + var cardResp = await ownerClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/cards", + new CreateCardDto(board.Id, col!.Id, "Edit-then-disconnect card", null, null, null)); + cardResp.StatusCode.Should().Be(HttpStatusCode.Created); + var card = await cardResp.Content.ReadFromJsonAsync(); + + // Second user who will disconnect + using var editorClient = _factory.CreateClient(); + var editor = await ApiTestHarness.AuthenticateAsync(editorClient, "race-disc-editor"); + var grant = await ownerClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/access", + new GrantAccessDto(board.Id, editor.UserId, UserRole.Editor)); + grant.StatusCode.Should().Be(HttpStatusCode.OK); + + // Owner joins and observes + var observerEvents = new EventCollector(); + await using var observer = SignalRTestHelper.CreateBoardsHubConnection(_factory, owner.Token); + observer.On("boardPresence", snapshot => observerEvents.Add(snapshot)); + await observer.StartAsync(); + await observer.InvokeAsync("JoinBoard", board.Id); + await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); + + // Editor joins and starts editing. + // We use 'await using' so the connection is always disposed (even if an + // assertion throws). The abrupt disconnect is simulated by disposing + // without calling LeaveBoard or SetEditingCard(null) first. + await using var editorConn = SignalRTestHelper.CreateBoardsHubConnection(_factory, editor.Token); + editorConn.On("boardPresence", _ => { }); + await editorConn.StartAsync(); + await editorConn.InvokeAsync("JoinBoard", board.Id); + await SignalRTestHelper.WaitForEventsAsync(observerEvents, 2); // join event + + observerEvents.Clear(); + await editorConn.InvokeAsync("SetEditingCard", board.Id, card!.Id); + + // Wait for editing presence update + var editingEvents = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); + var editorMember = editingEvents.Last().Members + .FirstOrDefault(m => m.UserId == editor.UserId); + editorMember.Should().NotBeNull("editor should be visible in presence"); + editorMember!.EditingCardId.Should().Be(card.Id, + "editor should show as editing the card"); + + // Abrupt disconnect (no LeaveBoard, no SetEditingCard(null)) + observerEvents.Clear(); + await editorConn.DisposeAsync(); + + // Owner should receive a snapshot without the editor + var afterDisconnect = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1, + TimeSpan.FromSeconds(5)); + afterDisconnect.Last().Members.Should().NotContain(m => m.UserId == editor.UserId, + "disconnected editor should be removed from presence"); + afterDisconnect.Last().Members.Should().ContainSingle(m => m.UserId == owner.UserId, + "only the owner should remain in presence after editor disconnects"); + } + + // ── Concurrent Webhook Delivery Creation ──────────────────────────────── + + /// + /// Scenario 16: Concurrent board mutations should each create webhook deliveries. + /// Multiple card operations fire concurrently on a board with an active + /// webhook subscription. Each mutation should produce its own delivery + /// record without duplicates or lost events. + /// + [Fact] + public async Task WebhookDelivery_ConcurrentBoardMutations_EachCreatesDeliveryRecord() + { + const int mutationCount = 5; + + using var client = _factory.CreateClient(); + await ApiTestHarness.AuthenticateAsync(client, "race-webhook-delivery"); + var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-board"); + + var colResp = await client.PostAsJsonAsync( + $"/api/boards/{board.Id}/columns", + new CreateColumnDto(board.Id, "Backlog", null, null)); + colResp.StatusCode.Should().Be(HttpStatusCode.Created); + var col = await colResp.Content.ReadFromJsonAsync(); + + // Create a webhook subscription on this board. + // NOTE: the endpoint URL is external-looking; delivery will fail at send time + // (no real server) but the delivery RECORD should be created. + var webhookResp = await client.PostAsJsonAsync( + $"/api/boards/{board.Id}/webhooks", + new CreateOutboundWebhookSubscriptionDto( + "https://example.com/webhook-receiver", + new List { "card.*" })); + webhookResp.StatusCode.Should().Be(HttpStatusCode.Created); + var webhookSub = await webhookResp.Content.ReadFromJsonAsync(); + webhookSub.Should().NotBeNull("webhook subscription should have been created"); + + // Create multiple cards concurrently — each should trigger a webhook delivery. + // Use Barrier for true synchronization instead of SemaphoreSlim. + using var barrier = new Barrier(mutationCount + 1); + var statusCodes = new ConcurrentBag(); + + var mutationTasks = Enumerable.Range(0, mutationCount).Select(async i => + { + using var raceClient = _factory.CreateClient(); + raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization; + barrier.SignalAndWait(TimeSpan.FromSeconds(10)); + var resp = await raceClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/cards", + new CreateCardDto(board.Id, col!.Id, $"Webhook card {i}", null, null, null)); + statusCodes.Add(resp.StatusCode); + }).ToArray(); + + barrier.SignalAndWait(TimeSpan.FromSeconds(10)); + await Task.WhenAll(mutationTasks); + + // All card creations should succeed + statusCodes.Should().AllSatisfy(s => + s.Should().Be(HttpStatusCode.Created), + "all concurrent card creations should succeed"); + + // Verify all cards were created (no duplicates, no losses) + var cardsResp = await client.GetAsync($"/api/boards/{board.Id}/cards"); + cardsResp.StatusCode.Should().Be(HttpStatusCode.OK); + var cards = await cardsResp.Content.ReadFromJsonAsync>(); + var webhookCards = cards!.Where(c => c.Title.StartsWith("Webhook card ")).ToList(); + webhookCards.Should().HaveCount(mutationCount, + "each concurrent mutation should create exactly one card (no duplicates or losses)"); + + // Verify all card titles are unique (no duplicate processing) + webhookCards.Select(c => c.Title).Distinct().Should().HaveCount(mutationCount, + "each card title should be unique, proving no duplicate processing"); + + // Verify that webhook delivery records were actually created in the database. + // Poll with a short timeout because the notifier enqueues deliveries + // asynchronously after the HTTP response returns. + using var scope = _factory.Services.CreateScope(); + var deliveryRepo = scope.ServiceProvider.GetRequiredService(); + + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10); + IReadOnlyList deliveries = []; + while (DateTimeOffset.UtcNow < deadline) + { + deliveries = await deliveryRepo.GetBySubscriptionAsync( + webhookSub!.Subscription.Id, limit: mutationCount + 5); + if (deliveries.Count >= mutationCount) + { + break; + } + + await Task.Delay(100); + } + + deliveries.Should().HaveCount(mutationCount, + $"each of the {mutationCount} card mutations should create exactly one webhook delivery record"); + deliveries.Select(d => d.Id).Distinct().Should().HaveCount(deliveries.Count, + "each delivery record should have a unique ID"); + } + + /// + /// Scenario 17: Concurrent webhook subscription creation on the same board. + /// Multiple webhook subscriptions created simultaneously should all succeed + /// with distinct IDs and secrets. + /// + [Fact] + public async Task WebhookSubscription_ConcurrentCreation_AllSucceedWithDistinctIds() + { + const int subscriptionCount = 3; + + using var client = _factory.CreateClient(); + await ApiTestHarness.AuthenticateAsync(client, "race-webhook-sub"); + var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-sub-board"); + + // Use Barrier for true synchronization instead of SemaphoreSlim + using var barrier = new Barrier(subscriptionCount + 1); + var results = new ConcurrentBag<(HttpStatusCode Status, OutboundWebhookSubscriptionSecretDto? Sub)>(); + + var tasks = Enumerable.Range(0, subscriptionCount).Select(async i => + { + using var raceClient = _factory.CreateClient(); + raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization; + barrier.SignalAndWait(TimeSpan.FromSeconds(10)); + var resp = await raceClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/webhooks", + new CreateOutboundWebhookSubscriptionDto( + $"https://example.com/webhook-{i}", + new List { "card.*" })); + var sub = resp.StatusCode == HttpStatusCode.Created + ? await resp.Content.ReadFromJsonAsync() + : null; + results.Add((resp.StatusCode, sub)); + }).ToArray(); + + barrier.SignalAndWait(TimeSpan.FromSeconds(10)); + await Task.WhenAll(tasks); + + // All should succeed + results.Select(r => r.Status).Should().AllSatisfy(s => + s.Should().Be(HttpStatusCode.Created), + "all concurrent webhook subscription creations should succeed"); + + // IDs should be distinct + var ids = results.Where(r => r.Sub != null).Select(r => r.Sub!.Subscription.Id).ToList(); + ids.Distinct().Should().HaveCount(subscriptionCount, + "each subscription should have a unique ID"); + + // Signing secrets should be distinct + var secrets = results.Where(r => r.Sub != null).Select(r => r.Sub!.SigningSecret).ToList(); + secrets.Distinct().Should().HaveCount(subscriptionCount, + "each subscription should have a unique signing secret"); + + // Verify via list endpoint: parse response and verify distinct IDs and count + var listResp = await client.GetAsync($"/api/boards/{board.Id}/webhooks"); + listResp.StatusCode.Should().Be(HttpStatusCode.OK); + var listedSubs = await listResp.Content.ReadFromJsonAsync>() + ?? throw new InvalidOperationException("list endpoint returned null subscription data"); + listedSubs.Should().HaveCountGreaterThanOrEqualTo(subscriptionCount, + $"list endpoint should return at least {subscriptionCount} subscriptions"); + listedSubs.Select(s => s.Id).Distinct().Should().HaveCountGreaterThanOrEqualTo(subscriptionCount, + "listed subscriptions should have distinct IDs"); + + // Cross-check: all IDs from creation should appear in the list + foreach (var createdId in ids) + { + listedSubs.Should().Contain(s => s.Id == createdId, + $"subscription {createdId} created concurrently should appear in the list endpoint"); + } + } }