From 9d99b125a10933213ff80aad65dd397bb9ea0642 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Fri, 10 Apr 2026 00:55:20 +0100 Subject: [PATCH 1/4] Add missing concurrency stress tests for SignalR presence and webhook delivery Adds 4 new scenarios to ConcurrencyRaceConditionStressTests: - Rapid join/leave presence stress with barrier-synchronized connections - Disconnect during edit clears editing state from presence - Concurrent board mutations each create webhook delivery records - Concurrent webhook subscription creation with distinct IDs/secrets --- .../ConcurrencyRaceConditionStressTests.cs | 297 ++++++++++++++++++ 1 file changed, 297 insertions(+) diff --git a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs index 4c1ccf92..c551fc4d 100644 --- a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs @@ -5,7 +5,9 @@ using FluentAssertions; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.SignalR.Client; using Taskdeck.Api.RateLimiting; +using Taskdeck.Api.Realtime; using Taskdeck.Api.Tests.Support; using Taskdeck.Application.DTOs; using Taskdeck.Domain.Entities; @@ -821,4 +823,299 @@ public async Task BoardCreation_ConcurrentMultiUser_NoCrossContamination() userBoards.Should().HaveCount(userCount, "all users should have created their boards successfully"); } + + // ── SignalR Presence Concurrency ──────────────────────────────────────── + + /// + /// Scenario 14: Rapid join/leave stress. + /// Multiple connections rapidly join and leave a board. + /// After all connections settle, the final presence snapshot should be + /// eventually consistent (only connections that remain joined are present). + /// + [Fact] + public async Task Presence_RapidJoinLeave_EventuallyConsistent() + { + const int connectionCount = 5; + + using var ownerClient = _factory.CreateClient(); + var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-presence-rapid"); + var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-rapid-board"); + + // Create users and grant access + var users = new List(); + for (var i = 0; i < connectionCount; i++) + { + using var c = _factory.CreateClient(); + var u = await ApiTestHarness.AuthenticateAsync(c, $"race-rapid-{i}"); + var grant = await ownerClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/access", + new GrantAccessDto(board.Id, u.UserId, UserRole.Editor)); + grant.StatusCode.Should().Be(HttpStatusCode.OK); + users.Add(u); + } + + // Owner observes presence events + var observerEvents = new EventCollector(); + await using var observer = SignalRTestHelper.CreateBoardsHubConnection(_factory, owner.Token); + observer.On("boardPresence", snapshot => observerEvents.Add(snapshot)); + await observer.StartAsync(); + await observer.InvokeAsync("JoinBoard", board.Id); + await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); + observerEvents.Clear(); + + // All users join simultaneously via barrier + var connections = new List(); + try + { + using var joinBarrier = new SemaphoreSlim(0, connectionCount); + var joinTasks = users.Select(async user => + { + var conn = SignalRTestHelper.CreateBoardsHubConnection(_factory, user.Token); + conn.On("boardPresence", _ => { }); + await conn.StartAsync(); + lock (connections) { connections.Add(conn); } + await joinBarrier.WaitAsync(); + await conn.InvokeAsync("JoinBoard", board.Id); + }).ToArray(); + + joinBarrier.Release(connectionCount); + await Task.WhenAll(joinTasks); + + // Wait for observer to see all joins + await SignalRTestHelper.WaitForEventsAsync(observerEvents, connectionCount, + TimeSpan.FromSeconds(10)); + + // All users should now be visible (plus owner) + var afterJoin = observerEvents.ToList().Last(); + afterJoin.Members.Should().HaveCount(connectionCount + 1, + "all joined users plus the observer owner should be present"); + + // Now have the first half leave rapidly + observerEvents.Clear(); + var leavingCount = connectionCount / 2; + using var leaveBarrier = new SemaphoreSlim(0, leavingCount); + var leaveTasks = connections.Take(leavingCount).Select(async conn => + { + await leaveBarrier.WaitAsync(); + await conn.InvokeAsync("LeaveBoard", board.Id); + }).ToArray(); + + leaveBarrier.Release(leavingCount); + await Task.WhenAll(leaveTasks); + + // Wait for leave events + await SignalRTestHelper.WaitForEventsAsync(observerEvents, leavingCount, + TimeSpan.FromSeconds(10)); + + // Final snapshot should show only the remaining connections + owner + var remaining = connectionCount - leavingCount; + var afterLeave = observerEvents.ToList().Last(); + afterLeave.Members.Should().HaveCount(remaining + 1, + $"after {leavingCount} leaves, {remaining} users + owner should remain"); + } + finally + { + foreach (var conn in connections) + { + await conn.DisposeAsync(); + } + } + } + + /// + /// Scenario 15: Disconnect during edit clears editing state. + /// A user sets an editing card, then their connection drops abruptly. + /// The presence snapshot should no longer include the editing state. + /// + [Fact] + public async Task Presence_DisconnectDuringEdit_ClearsEditingState() + { + using var ownerClient = _factory.CreateClient(); + var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-disc-edit"); + var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-disc-edit-board"); + + // Create a column and card + var colResp = await ownerClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/columns", + new CreateColumnDto(board.Id, "Backlog", null, null)); + colResp.StatusCode.Should().Be(HttpStatusCode.Created); + var col = await colResp.Content.ReadFromJsonAsync(); + + var cardResp = await ownerClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/cards", + new CreateCardDto(board.Id, col!.Id, "Edit-then-disconnect card", null, null, null)); + cardResp.StatusCode.Should().Be(HttpStatusCode.Created); + var card = await cardResp.Content.ReadFromJsonAsync(); + + // Second user who will disconnect + using var editorClient = _factory.CreateClient(); + var editor = await ApiTestHarness.AuthenticateAsync(editorClient, "race-disc-editor"); + var grant = await ownerClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/access", + new GrantAccessDto(board.Id, editor.UserId, UserRole.Editor)); + grant.StatusCode.Should().Be(HttpStatusCode.OK); + + // Owner joins and observes + var observerEvents = new EventCollector(); + await using var observer = SignalRTestHelper.CreateBoardsHubConnection(_factory, owner.Token); + observer.On("boardPresence", snapshot => observerEvents.Add(snapshot)); + await observer.StartAsync(); + await observer.InvokeAsync("JoinBoard", board.Id); + await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); + + // Editor joins and starts editing + var editorConn = SignalRTestHelper.CreateBoardsHubConnection(_factory, editor.Token); + editorConn.On("boardPresence", _ => { }); + await editorConn.StartAsync(); + await editorConn.InvokeAsync("JoinBoard", board.Id); + await SignalRTestHelper.WaitForEventsAsync(observerEvents, 2); // join event + + observerEvents.Clear(); + await editorConn.InvokeAsync("SetEditingCard", board.Id, card!.Id); + + // Wait for editing presence update + var editingEvents = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); + var editorMember = editingEvents.Last().Members + .FirstOrDefault(m => m.UserId == editor.UserId); + editorMember.Should().NotBeNull("editor should be visible in presence"); + editorMember!.EditingCardId.Should().Be(card.Id, + "editor should show as editing the card"); + + // Abrupt disconnect (no LeaveBoard, no SetEditingCard(null)) + observerEvents.Clear(); + await editorConn.DisposeAsync(); + + // Owner should receive a snapshot without the editor + var afterDisconnect = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1, + TimeSpan.FromSeconds(5)); + afterDisconnect.Last().Members.Should().NotContain(m => m.UserId == editor.UserId, + "disconnected editor should be removed from presence"); + afterDisconnect.Last().Members.Should().ContainSingle(m => m.UserId == owner.UserId, + "only the owner should remain in presence after editor disconnects"); + } + + // ── Concurrent Webhook Delivery Creation ──────────────────────────────── + + /// + /// Scenario 16: Concurrent board mutations should each create webhook deliveries. + /// Multiple card operations fire concurrently on a board with an active + /// webhook subscription. Each mutation should produce its own delivery + /// record without duplicates or lost events. + /// + [Fact] + public async Task WebhookDelivery_ConcurrentBoardMutations_EachCreatesDeliveryRecord() + { + const int mutationCount = 5; + + using var client = _factory.CreateClient(); + await ApiTestHarness.AuthenticateAsync(client, "race-webhook-delivery"); + var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-board"); + + var colResp = await client.PostAsJsonAsync( + $"/api/boards/{board.Id}/columns", + new CreateColumnDto(board.Id, "Backlog", null, null)); + colResp.StatusCode.Should().Be(HttpStatusCode.Created); + var col = await colResp.Content.ReadFromJsonAsync(); + + // Create a webhook subscription on this board. + // NOTE: the endpoint URL is external-looking; delivery will fail at send time + // (no real server) but the delivery RECORD should be created. + var webhookResp = await client.PostAsJsonAsync( + $"/api/boards/{board.Id}/webhooks", + new CreateOutboundWebhookSubscriptionDto( + "https://example.com/webhook-receiver", + new List { "card.*" })); + webhookResp.StatusCode.Should().Be(HttpStatusCode.Created); + + // Create multiple cards concurrently — each should trigger a webhook delivery + using var barrier = new SemaphoreSlim(0, mutationCount); + var statusCodes = new ConcurrentBag(); + + var mutationTasks = Enumerable.Range(0, mutationCount).Select(async i => + { + using var raceClient = _factory.CreateClient(); + raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization; + await barrier.WaitAsync(); + var resp = await raceClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/cards", + new CreateCardDto(board.Id, col!.Id, $"Webhook card {i}", null, null, null)); + statusCodes.Add(resp.StatusCode); + }).ToArray(); + + barrier.Release(mutationCount); + await Task.WhenAll(mutationTasks); + + // All card creations should succeed + statusCodes.Should().AllSatisfy(s => + s.Should().Be(HttpStatusCode.Created), + "all concurrent card creations should succeed"); + + // Verify all cards were created (no duplicates, no losses) + var cardsResp = await client.GetAsync($"/api/boards/{board.Id}/cards"); + cardsResp.StatusCode.Should().Be(HttpStatusCode.OK); + var cards = await cardsResp.Content.ReadFromJsonAsync>(); + var webhookCards = cards!.Where(c => c.Title.StartsWith("Webhook card ")).ToList(); + webhookCards.Should().HaveCount(mutationCount, + "each concurrent mutation should create exactly one card (no duplicates or losses)"); + + // Verify all card titles are unique (no duplicate processing) + webhookCards.Select(c => c.Title).Distinct().Should().HaveCount(mutationCount, + "each card title should be unique, proving no duplicate processing"); + } + + /// + /// Scenario 17: Concurrent webhook subscription creation on the same board. + /// Multiple webhook subscriptions created simultaneously should all succeed + /// with distinct IDs and secrets. + /// + [Fact] + public async Task WebhookSubscription_ConcurrentCreation_AllSucceedWithDistinctIds() + { + const int subscriptionCount = 3; + + using var client = _factory.CreateClient(); + await ApiTestHarness.AuthenticateAsync(client, "race-webhook-sub"); + var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-sub-board"); + + using var barrier = new SemaphoreSlim(0, subscriptionCount); + var results = new ConcurrentBag<(HttpStatusCode Status, OutboundWebhookSubscriptionSecretDto? Sub)>(); + + var tasks = Enumerable.Range(0, subscriptionCount).Select(async i => + { + using var raceClient = _factory.CreateClient(); + raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization; + await barrier.WaitAsync(); + var resp = await raceClient.PostAsJsonAsync( + $"/api/boards/{board.Id}/webhooks", + new CreateOutboundWebhookSubscriptionDto( + $"https://example.com/webhook-{i}", + new List { "card.*" })); + var sub = resp.StatusCode == HttpStatusCode.Created + ? await resp.Content.ReadFromJsonAsync() + : null; + results.Add((resp.StatusCode, sub)); + }).ToArray(); + + barrier.Release(subscriptionCount); + await Task.WhenAll(tasks); + + // All should succeed + results.Select(r => r.Status).Should().AllSatisfy(s => + s.Should().Be(HttpStatusCode.Created), + "all concurrent webhook subscription creations should succeed"); + + // IDs should be distinct + var ids = results.Where(r => r.Sub != null).Select(r => r.Sub!.Subscription.Id).ToList(); + ids.Distinct().Should().HaveCount(subscriptionCount, + "each subscription should have a unique ID"); + + // Signing secrets should be distinct + var secrets = results.Where(r => r.Sub != null).Select(r => r.Sub!.SigningSecret).ToList(); + secrets.Distinct().Should().HaveCount(subscriptionCount, + "each subscription should have a unique signing secret"); + + // Verify via list endpoint + var listResp = await client.GetAsync($"/api/boards/{board.Id}/webhooks"); + listResp.StatusCode.Should().Be(HttpStatusCode.OK); + } } From 883cfc8f38475eb65d3fcf25b1dfd4a70e4a7154 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Fri, 10 Apr 2026 00:57:33 +0100 Subject: [PATCH 2/4] Fix resource leak in disconnect-during-edit test Wrap editorConn in try/finally to ensure disposal even if assertions fail before the explicit DisposeAsync call. --- .../ConcurrencyRaceConditionStressTests.cs | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs index c551fc4d..f8d525fc 100644 --- a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs @@ -963,27 +963,35 @@ public async Task Presence_DisconnectDuringEdit_ClearsEditingState() await observer.InvokeAsync("JoinBoard", board.Id); await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); - // Editor joins and starts editing + // Editor joins and starts editing. + // Not using 'await using' because we need to control when disposal happens + // to simulate an abrupt disconnect. The try/finally ensures cleanup on failure. var editorConn = SignalRTestHelper.CreateBoardsHubConnection(_factory, editor.Token); - editorConn.On("boardPresence", _ => { }); - await editorConn.StartAsync(); - await editorConn.InvokeAsync("JoinBoard", board.Id); - await SignalRTestHelper.WaitForEventsAsync(observerEvents, 2); // join event + try + { + editorConn.On("boardPresence", _ => { }); + await editorConn.StartAsync(); + await editorConn.InvokeAsync("JoinBoard", board.Id); + await SignalRTestHelper.WaitForEventsAsync(observerEvents, 2); // join event - observerEvents.Clear(); - await editorConn.InvokeAsync("SetEditingCard", board.Id, card!.Id); + observerEvents.Clear(); + await editorConn.InvokeAsync("SetEditingCard", board.Id, card!.Id); - // Wait for editing presence update - var editingEvents = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); - var editorMember = editingEvents.Last().Members - .FirstOrDefault(m => m.UserId == editor.UserId); - editorMember.Should().NotBeNull("editor should be visible in presence"); - editorMember!.EditingCardId.Should().Be(card.Id, - "editor should show as editing the card"); + // Wait for editing presence update + var editingEvents = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); + var editorMember = editingEvents.Last().Members + .FirstOrDefault(m => m.UserId == editor.UserId); + editorMember.Should().NotBeNull("editor should be visible in presence"); + editorMember!.EditingCardId.Should().Be(card.Id, + "editor should show as editing the card"); - // Abrupt disconnect (no LeaveBoard, no SetEditingCard(null)) - observerEvents.Clear(); - await editorConn.DisposeAsync(); + // Abrupt disconnect (no LeaveBoard, no SetEditingCard(null)) + observerEvents.Clear(); + } + finally + { + await editorConn.DisposeAsync(); + } // Owner should receive a snapshot without the editor var afterDisconnect = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1, From 3edf94c71e80ea3eb107530e6236c8ceb4f1bb81 Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Sat, 11 Apr 2026 23:56:21 +0100 Subject: [PATCH 3/4] Fix concurrency stress test review findings from PR #814 Address all bot review comments and adversarial review findings: 1. Replace SemaphoreSlim barriers with Barrier class for true synchronization in Scenarios 14, 16, 17. Barrier guarantees all participants arrive before any proceed, unlike SemaphoreSlim which can release permits before all tasks reach WaitAsync(). 2. Add WaitForPresenceCountAsync polling helper to avoid flaky assertions. Instead of checking .Last() snapshot (which may not reflect settled state), poll until a snapshot with the expected member count appears. 3. Wrap editorConn in await using for Scenario 15 to prevent connection leaks on assertion failure. The abrupt disconnect is simulated by explicit DisposeAsync() before the final assertions. 4. Add actual webhook delivery record verification in Scenario 16. The test name says "EachCreatesDeliveryRecord" but previously only verified cards were created. Now queries the delivery repository via DI to confirm delivery records exist with distinct IDs. 5. Parse and verify list endpoint response in Scenario 17. Previously only checked HTTP 200 status. Now deserializes the response, checks subscription count, verifies distinct IDs, and cross-checks that all concurrently-created subscription IDs appear in the list. 6. Reuse single HttpClient for user setup loop in Scenario 14 instead of creating one per iteration. --- .../ConcurrencyRaceConditionStressTests.cs | 178 ++++++++++++------ 1 file changed, 122 insertions(+), 56 deletions(-) diff --git a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs index f8d525fc..a3589e22 100644 --- a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs @@ -6,10 +6,12 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.AspNetCore.SignalR.Client; +using Microsoft.Extensions.DependencyInjection; using Taskdeck.Api.RateLimiting; using Taskdeck.Api.Realtime; using Taskdeck.Api.Tests.Support; using Taskdeck.Application.DTOs; +using Taskdeck.Application.Interfaces; using Taskdeck.Domain.Entities; using Taskdeck.Domain.Enums; using Xunit; @@ -24,7 +26,7 @@ namespace Taskdeck.Api.Tests; /// - Rate limiting under load (burst beyond limit, cross-user isolation under load) /// /// Uses Task.WhenAll with multiple HttpClient instances for HTTP-level concurrency. -/// Uses SemaphoreSlim barriers to ensure truly simultaneous execution. +/// Uses Barrier / SemaphoreSlim to coordinate truly simultaneous execution. /// /// NOTE: SQLite uses file-level locking for writes, so true write-contention races /// may serialize at the database level. These tests validate the application-layer @@ -826,6 +828,35 @@ public async Task BoardCreation_ConcurrentMultiUser_NoCrossContamination() // ── SignalR Presence Concurrency ──────────────────────────────────────── + /// + /// Polls the observer's event collector until a snapshot with the expected + /// member count appears, or the timeout elapses. This avoids flakiness + /// from checking only the last event, which may not reflect the settled state. + /// + private static async Task WaitForPresenceCountAsync( + EventCollector events, + int expectedMemberCount, + TimeSpan? timeout = null) + { + var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(10); + var deadline = DateTimeOffset.UtcNow + effectiveTimeout; + while (DateTimeOffset.UtcNow < deadline) + { + var snapshot = events.ToList().LastOrDefault(); + if (snapshot is not null && snapshot.Members.Count == expectedMemberCount) + { + return snapshot; + } + + await Task.Delay(50); + } + + var last = events.ToList().LastOrDefault(); + var actualCount = last?.Members.Count ?? 0; + throw new TimeoutException( + $"Expected presence snapshot with {expectedMemberCount} members but last snapshot had {actualCount} within {effectiveTimeout.TotalSeconds}s."); + } + /// /// Scenario 14: Rapid join/leave stress. /// Multiple connections rapidly join and leave a board. @@ -841,12 +872,12 @@ public async Task Presence_RapidJoinLeave_EventuallyConsistent() var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-presence-rapid"); var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-rapid-board"); - // Create users and grant access + // Create users and grant access — reuse a single HttpClient for setup + using var setupClient = _factory.CreateClient(); var users = new List(); for (var i = 0; i < connectionCount; i++) { - using var c = _factory.CreateClient(); - var u = await ApiTestHarness.AuthenticateAsync(c, $"race-rapid-{i}"); + var u = await ApiTestHarness.AuthenticateAsync(setupClient, $"race-rapid-{i}"); var grant = await ownerClient.PostAsJsonAsync( $"/api/boards/{board.Id}/access", new GrantAccessDto(board.Id, u.UserId, UserRole.Editor)); @@ -863,53 +894,51 @@ public async Task Presence_RapidJoinLeave_EventuallyConsistent() await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); observerEvents.Clear(); - // All users join simultaneously via barrier + // All users join simultaneously via Barrier for true synchronization. + // Unlike SemaphoreSlim, Barrier ensures all participants reach the + // barrier point before any of them proceed past it. var connections = new List(); try { - using var joinBarrier = new SemaphoreSlim(0, connectionCount); + using var joinBarrier = new Barrier(connectionCount + 1); var joinTasks = users.Select(async user => { var conn = SignalRTestHelper.CreateBoardsHubConnection(_factory, user.Token); conn.On("boardPresence", _ => { }); await conn.StartAsync(); lock (connections) { connections.Add(conn); } - await joinBarrier.WaitAsync(); + joinBarrier.SignalAndWait(TimeSpan.FromSeconds(10)); await conn.InvokeAsync("JoinBoard", board.Id); }).ToArray(); - joinBarrier.Release(connectionCount); + joinBarrier.SignalAndWait(TimeSpan.FromSeconds(10)); await Task.WhenAll(joinTasks); - // Wait for observer to see all joins - await SignalRTestHelper.WaitForEventsAsync(observerEvents, connectionCount, - TimeSpan.FromSeconds(10)); - - // All users should now be visible (plus owner) - var afterJoin = observerEvents.ToList().Last(); + // Poll until the observer snapshot settles at the expected member count + // (all joined users plus the owner). This avoids flakiness from + // intermediate snapshots that don't yet reflect all joins. + var afterJoin = await WaitForPresenceCountAsync( + observerEvents, connectionCount + 1, TimeSpan.FromSeconds(10)); afterJoin.Members.Should().HaveCount(connectionCount + 1, "all joined users plus the observer owner should be present"); // Now have the first half leave rapidly observerEvents.Clear(); var leavingCount = connectionCount / 2; - using var leaveBarrier = new SemaphoreSlim(0, leavingCount); + using var leaveBarrier = new Barrier(leavingCount + 1); var leaveTasks = connections.Take(leavingCount).Select(async conn => { - await leaveBarrier.WaitAsync(); + leaveBarrier.SignalAndWait(TimeSpan.FromSeconds(10)); await conn.InvokeAsync("LeaveBoard", board.Id); }).ToArray(); - leaveBarrier.Release(leavingCount); + leaveBarrier.SignalAndWait(TimeSpan.FromSeconds(10)); await Task.WhenAll(leaveTasks); - // Wait for leave events - await SignalRTestHelper.WaitForEventsAsync(observerEvents, leavingCount, - TimeSpan.FromSeconds(10)); - - // Final snapshot should show only the remaining connections + owner + // Poll until the snapshot settles at the expected remaining count var remaining = connectionCount - leavingCount; - var afterLeave = observerEvents.ToList().Last(); + var afterLeave = await WaitForPresenceCountAsync( + observerEvents, remaining + 1, TimeSpan.FromSeconds(10)); afterLeave.Members.Should().HaveCount(remaining + 1, $"after {leavingCount} leaves, {remaining} users + owner should remain"); } @@ -964,34 +993,29 @@ public async Task Presence_DisconnectDuringEdit_ClearsEditingState() await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); // Editor joins and starts editing. - // Not using 'await using' because we need to control when disposal happens - // to simulate an abrupt disconnect. The try/finally ensures cleanup on failure. - var editorConn = SignalRTestHelper.CreateBoardsHubConnection(_factory, editor.Token); - try - { - editorConn.On("boardPresence", _ => { }); - await editorConn.StartAsync(); - await editorConn.InvokeAsync("JoinBoard", board.Id); - await SignalRTestHelper.WaitForEventsAsync(observerEvents, 2); // join event + // We use 'await using' so the connection is always disposed (even if an + // assertion throws). The abrupt disconnect is simulated by disposing + // without calling LeaveBoard or SetEditingCard(null) first. + await using var editorConn = SignalRTestHelper.CreateBoardsHubConnection(_factory, editor.Token); + editorConn.On("boardPresence", _ => { }); + await editorConn.StartAsync(); + await editorConn.InvokeAsync("JoinBoard", board.Id); + await SignalRTestHelper.WaitForEventsAsync(observerEvents, 2); // join event - observerEvents.Clear(); - await editorConn.InvokeAsync("SetEditingCard", board.Id, card!.Id); + observerEvents.Clear(); + await editorConn.InvokeAsync("SetEditingCard", board.Id, card!.Id); - // Wait for editing presence update - var editingEvents = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); - var editorMember = editingEvents.Last().Members - .FirstOrDefault(m => m.UserId == editor.UserId); - editorMember.Should().NotBeNull("editor should be visible in presence"); - editorMember!.EditingCardId.Should().Be(card.Id, - "editor should show as editing the card"); + // Wait for editing presence update + var editingEvents = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1); + var editorMember = editingEvents.Last().Members + .FirstOrDefault(m => m.UserId == editor.UserId); + editorMember.Should().NotBeNull("editor should be visible in presence"); + editorMember!.EditingCardId.Should().Be(card.Id, + "editor should show as editing the card"); - // Abrupt disconnect (no LeaveBoard, no SetEditingCard(null)) - observerEvents.Clear(); - } - finally - { - await editorConn.DisposeAsync(); - } + // Abrupt disconnect (no LeaveBoard, no SetEditingCard(null)) + observerEvents.Clear(); + await editorConn.DisposeAsync(); // Owner should receive a snapshot without the editor var afterDisconnect = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1, @@ -1034,23 +1058,26 @@ public async Task WebhookDelivery_ConcurrentBoardMutations_EachCreatesDeliveryRe "https://example.com/webhook-receiver", new List { "card.*" })); webhookResp.StatusCode.Should().Be(HttpStatusCode.Created); + var webhookSub = await webhookResp.Content.ReadFromJsonAsync(); + webhookSub.Should().NotBeNull("webhook subscription should have been created"); - // Create multiple cards concurrently — each should trigger a webhook delivery - using var barrier = new SemaphoreSlim(0, mutationCount); + // Create multiple cards concurrently — each should trigger a webhook delivery. + // Use Barrier for true synchronization instead of SemaphoreSlim. + using var barrier = new Barrier(mutationCount + 1); var statusCodes = new ConcurrentBag(); var mutationTasks = Enumerable.Range(0, mutationCount).Select(async i => { using var raceClient = _factory.CreateClient(); raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization; - await barrier.WaitAsync(); + barrier.SignalAndWait(TimeSpan.FromSeconds(10)); var resp = await raceClient.PostAsJsonAsync( $"/api/boards/{board.Id}/cards", new CreateCardDto(board.Id, col!.Id, $"Webhook card {i}", null, null, null)); statusCodes.Add(resp.StatusCode); }).ToArray(); - barrier.Release(mutationCount); + barrier.SignalAndWait(TimeSpan.FromSeconds(10)); await Task.WhenAll(mutationTasks); // All card creations should succeed @@ -1069,6 +1096,31 @@ public async Task WebhookDelivery_ConcurrentBoardMutations_EachCreatesDeliveryRe // Verify all card titles are unique (no duplicate processing) webhookCards.Select(c => c.Title).Distinct().Should().HaveCount(mutationCount, "each card title should be unique, proving no duplicate processing"); + + // Verify that webhook delivery records were actually created in the database. + // Poll with a short timeout because the notifier enqueues deliveries + // asynchronously after the HTTP response returns. + using var scope = _factory.Services.CreateScope(); + var deliveryRepo = scope.ServiceProvider.GetRequiredService(); + + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10); + IReadOnlyList deliveries = []; + while (DateTimeOffset.UtcNow < deadline) + { + deliveries = await deliveryRepo.GetBySubscriptionAsync( + webhookSub!.Subscription.Id, limit: mutationCount + 5); + if (deliveries.Count >= mutationCount) + { + break; + } + + await Task.Delay(100); + } + + deliveries.Should().HaveCountGreaterThanOrEqualTo(mutationCount, + $"each of the {mutationCount} card mutations should have created a webhook delivery record"); + deliveries.Select(d => d.Id).Distinct().Should().HaveCount(deliveries.Count, + "each delivery record should have a unique ID"); } /// @@ -1085,14 +1137,15 @@ public async Task WebhookSubscription_ConcurrentCreation_AllSucceedWithDistinctI await ApiTestHarness.AuthenticateAsync(client, "race-webhook-sub"); var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-sub-board"); - using var barrier = new SemaphoreSlim(0, subscriptionCount); + // Use Barrier for true synchronization instead of SemaphoreSlim + using var barrier = new Barrier(subscriptionCount + 1); var results = new ConcurrentBag<(HttpStatusCode Status, OutboundWebhookSubscriptionSecretDto? Sub)>(); var tasks = Enumerable.Range(0, subscriptionCount).Select(async i => { using var raceClient = _factory.CreateClient(); raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization; - await barrier.WaitAsync(); + barrier.SignalAndWait(TimeSpan.FromSeconds(10)); var resp = await raceClient.PostAsJsonAsync( $"/api/boards/{board.Id}/webhooks", new CreateOutboundWebhookSubscriptionDto( @@ -1104,7 +1157,7 @@ public async Task WebhookSubscription_ConcurrentCreation_AllSucceedWithDistinctI results.Add((resp.StatusCode, sub)); }).ToArray(); - barrier.Release(subscriptionCount); + barrier.SignalAndWait(TimeSpan.FromSeconds(10)); await Task.WhenAll(tasks); // All should succeed @@ -1122,8 +1175,21 @@ public async Task WebhookSubscription_ConcurrentCreation_AllSucceedWithDistinctI secrets.Distinct().Should().HaveCount(subscriptionCount, "each subscription should have a unique signing secret"); - // Verify via list endpoint + // Verify via list endpoint: parse response and verify distinct IDs and count var listResp = await client.GetAsync($"/api/boards/{board.Id}/webhooks"); listResp.StatusCode.Should().Be(HttpStatusCode.OK); + var listedSubs = await listResp.Content.ReadFromJsonAsync>() + ?? throw new InvalidOperationException("list endpoint returned null subscription data"); + listedSubs.Should().HaveCountGreaterThanOrEqualTo(subscriptionCount, + $"list endpoint should return at least {subscriptionCount} subscriptions"); + listedSubs.Select(s => s.Id).Distinct().Should().HaveCountGreaterThanOrEqualTo(subscriptionCount, + "listed subscriptions should have distinct IDs"); + + // Cross-check: all IDs from creation should appear in the list + foreach (var createdId in ids) + { + listedSubs.Should().Contain(s => s.Id == createdId, + $"subscription {createdId} created concurrently should appear in the list endpoint"); + } } } From 8d608449f1a585dc429d4d8c18f29ef55a9f926a Mon Sep 17 00:00:00 2001 From: Chris0Jeky Date: Sun, 12 Apr 2026 00:39:57 +0100 Subject: [PATCH 4/4] Tighten webhook concurrency delivery assertion --- .../Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs index a3589e22..9cc84a7a 100644 --- a/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs +++ b/backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs @@ -1117,8 +1117,8 @@ public async Task WebhookDelivery_ConcurrentBoardMutations_EachCreatesDeliveryRe await Task.Delay(100); } - deliveries.Should().HaveCountGreaterThanOrEqualTo(mutationCount, - $"each of the {mutationCount} card mutations should have created a webhook delivery record"); + deliveries.Should().HaveCount(mutationCount, + $"each of the {mutationCount} card mutations should create exactly one webhook delivery record"); deliveries.Select(d => d.Id).Distinct().Should().HaveCount(deliveries.Count, "each delivery record should have a unique ID"); }