Skip to content

Commit 473f1b4

Browse files
Chris0JekyChris0Jeky
authored
Merge pull request #814 from Chris0Jeky/test/concurrency-race-condition-stress
TST-38: Add missing concurrency stress tests for presence and webhooks
2 parents 1b6c210 + 8d60844 commit 473f1b4

File tree

1 file changed

+372
-1
lines changed

1 file changed

+372
-1
lines changed

‎backend/tests/Taskdeck.Api.Tests/ConcurrencyRaceConditionStressTests.cs‎

Lines changed: 372 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,13 @@
55
using FluentAssertions;
66
using Microsoft.AspNetCore.Hosting;
77
using Microsoft.AspNetCore.Mvc.Testing;
8+
using Microsoft.AspNetCore.SignalR.Client;
9+
using Microsoft.Extensions.DependencyInjection;
810
using Taskdeck.Api.RateLimiting;
11+
using Taskdeck.Api.Realtime;
912
using Taskdeck.Api.Tests.Support;
1013
using Taskdeck.Application.DTOs;
14+
using Taskdeck.Application.Interfaces;
1115
using Taskdeck.Domain.Entities;
1216
using Taskdeck.Domain.Enums;
1317
using Xunit;
@@ -22,7 +26,7 @@ namespace Taskdeck.Api.Tests;
2226
/// - Rate limiting under load (burst beyond limit, cross-user isolation under load)
2327
///
2428
/// Uses Task.WhenAll with multiple HttpClient instances for HTTP-level concurrency.
25-
/// Uses SemaphoreSlim barriers to ensure truly simultaneous execution.
29+
/// Uses Barrier / SemaphoreSlim to coordinate truly simultaneous execution.
2630
///
2731
/// NOTE: SQLite uses file-level locking for writes, so true write-contention races
2832
/// may serialize at the database level. These tests validate the application-layer
@@ -821,4 +825,371 @@ public async Task BoardCreation_ConcurrentMultiUser_NoCrossContamination()
821825
userBoards.Should().HaveCount(userCount,
822826
"all users should have created their boards successfully");
823827
}
828+
829+
// ── SignalR Presence Concurrency ────────────────────────────────────────
830+
831+
/// <summary>
832+
/// Polls the observer's event collector until a snapshot with the expected
833+
/// member count appears, or the timeout elapses. This avoids flakiness
834+
/// from checking only the last event, which may not reflect the settled state.
835+
/// </summary>
836+
private static async Task<BoardPresenceSnapshot> WaitForPresenceCountAsync(
837+
EventCollector<BoardPresenceSnapshot> events,
838+
int expectedMemberCount,
839+
TimeSpan? timeout = null)
840+
{
841+
var effectiveTimeout = timeout ?? TimeSpan.FromSeconds(10);
842+
var deadline = DateTimeOffset.UtcNow + effectiveTimeout;
843+
while (DateTimeOffset.UtcNow < deadline)
844+
{
845+
var snapshot = events.ToList().LastOrDefault();
846+
if (snapshot is not null && snapshot.Members.Count == expectedMemberCount)
847+
{
848+
return snapshot;
849+
}
850+
851+
await Task.Delay(50);
852+
}
853+
854+
var last = events.ToList().LastOrDefault();
855+
var actualCount = last?.Members.Count ?? 0;
856+
throw new TimeoutException(
857+
$"Expected presence snapshot with {expectedMemberCount} members but last snapshot had {actualCount} within {effectiveTimeout.TotalSeconds}s.");
858+
}
859+
860+
/// <summary>
861+
/// Scenario 14: Rapid join/leave stress.
862+
/// Multiple connections rapidly join and leave a board.
863+
/// After all connections settle, the final presence snapshot should be
864+
/// eventually consistent (only connections that remain joined are present).
865+
/// </summary>
866+
[Fact]
867+
public async Task Presence_RapidJoinLeave_EventuallyConsistent()
868+
{
869+
const int connectionCount = 5;
870+
871+
using var ownerClient = _factory.CreateClient();
872+
var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-presence-rapid");
873+
var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-rapid-board");
874+
875+
// Create users and grant access — reuse a single HttpClient for setup
876+
using var setupClient = _factory.CreateClient();
877+
var users = new List<TestUserContext>();
878+
for (var i = 0; i < connectionCount; i++)
879+
{
880+
var u = await ApiTestHarness.AuthenticateAsync(setupClient, $"race-rapid-{i}");
881+
var grant = await ownerClient.PostAsJsonAsync(
882+
$"/api/boards/{board.Id}/access",
883+
new GrantAccessDto(board.Id, u.UserId, UserRole.Editor));
884+
grant.StatusCode.Should().Be(HttpStatusCode.OK);
885+
users.Add(u);
886+
}
887+
888+
// Owner observes presence events
889+
var observerEvents = new EventCollector<BoardPresenceSnapshot>();
890+
await using var observer = SignalRTestHelper.CreateBoardsHubConnection(_factory, owner.Token);
891+
observer.On<BoardPresenceSnapshot>("boardPresence", snapshot => observerEvents.Add(snapshot));
892+
await observer.StartAsync();
893+
await observer.InvokeAsync("JoinBoard", board.Id);
894+
await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1);
895+
observerEvents.Clear();
896+
897+
// All users join simultaneously via Barrier for true synchronization.
898+
// Unlike SemaphoreSlim, Barrier ensures all participants reach the
899+
// barrier point before any of them proceed past it.
900+
var connections = new List<HubConnection>();
901+
try
902+
{
903+
using var joinBarrier = new Barrier(connectionCount + 1);
904+
var joinTasks = users.Select(async user =>
905+
{
906+
var conn = SignalRTestHelper.CreateBoardsHubConnection(_factory, user.Token);
907+
conn.On<BoardPresenceSnapshot>("boardPresence", _ => { });
908+
await conn.StartAsync();
909+
lock (connections) { connections.Add(conn); }
910+
joinBarrier.SignalAndWait(TimeSpan.FromSeconds(10));
911+
await conn.InvokeAsync("JoinBoard", board.Id);
912+
}).ToArray();
913+
914+
joinBarrier.SignalAndWait(TimeSpan.FromSeconds(10));
915+
await Task.WhenAll(joinTasks);
916+
917+
// Poll until the observer snapshot settles at the expected member count
918+
// (all joined users plus the owner). This avoids flakiness from
919+
// intermediate snapshots that don't yet reflect all joins.
920+
var afterJoin = await WaitForPresenceCountAsync(
921+
observerEvents, connectionCount + 1, TimeSpan.FromSeconds(10));
922+
afterJoin.Members.Should().HaveCount(connectionCount + 1,
923+
"all joined users plus the observer owner should be present");
924+
925+
// Now have the first half leave rapidly
926+
observerEvents.Clear();
927+
var leavingCount = connectionCount / 2;
928+
using var leaveBarrier = new Barrier(leavingCount + 1);
929+
var leaveTasks = connections.Take(leavingCount).Select(async conn =>
930+
{
931+
leaveBarrier.SignalAndWait(TimeSpan.FromSeconds(10));
932+
await conn.InvokeAsync("LeaveBoard", board.Id);
933+
}).ToArray();
934+
935+
leaveBarrier.SignalAndWait(TimeSpan.FromSeconds(10));
936+
await Task.WhenAll(leaveTasks);
937+
938+
// Poll until the snapshot settles at the expected remaining count
939+
var remaining = connectionCount - leavingCount;
940+
var afterLeave = await WaitForPresenceCountAsync(
941+
observerEvents, remaining + 1, TimeSpan.FromSeconds(10));
942+
afterLeave.Members.Should().HaveCount(remaining + 1,
943+
$"after {leavingCount} leaves, {remaining} users + owner should remain");
944+
}
945+
finally
946+
{
947+
foreach (var conn in connections)
948+
{
949+
await conn.DisposeAsync();
950+
}
951+
}
952+
}
953+
954+
/// <summary>
955+
/// Scenario 15: Disconnect during edit clears editing state.
956+
/// A user sets an editing card, then their connection drops abruptly.
957+
/// The presence snapshot should no longer include the editing state.
958+
/// </summary>
959+
[Fact]
960+
public async Task Presence_DisconnectDuringEdit_ClearsEditingState()
961+
{
962+
using var ownerClient = _factory.CreateClient();
963+
var owner = await ApiTestHarness.AuthenticateAsync(ownerClient, "race-disc-edit");
964+
var board = await ApiTestHarness.CreateBoardAsync(ownerClient, "race-disc-edit-board");
965+
966+
// Create a column and card
967+
var colResp = await ownerClient.PostAsJsonAsync(
968+
$"/api/boards/{board.Id}/columns",
969+
new CreateColumnDto(board.Id, "Backlog", null, null));
970+
colResp.StatusCode.Should().Be(HttpStatusCode.Created);
971+
var col = await colResp.Content.ReadFromJsonAsync<ColumnDto>();
972+
973+
var cardResp = await ownerClient.PostAsJsonAsync(
974+
$"/api/boards/{board.Id}/cards",
975+
new CreateCardDto(board.Id, col!.Id, "Edit-then-disconnect card", null, null, null));
976+
cardResp.StatusCode.Should().Be(HttpStatusCode.Created);
977+
var card = await cardResp.Content.ReadFromJsonAsync<CardDto>();
978+
979+
// Second user who will disconnect
980+
using var editorClient = _factory.CreateClient();
981+
var editor = await ApiTestHarness.AuthenticateAsync(editorClient, "race-disc-editor");
982+
var grant = await ownerClient.PostAsJsonAsync(
983+
$"/api/boards/{board.Id}/access",
984+
new GrantAccessDto(board.Id, editor.UserId, UserRole.Editor));
985+
grant.StatusCode.Should().Be(HttpStatusCode.OK);
986+
987+
// Owner joins and observes
988+
var observerEvents = new EventCollector<BoardPresenceSnapshot>();
989+
await using var observer = SignalRTestHelper.CreateBoardsHubConnection(_factory, owner.Token);
990+
observer.On<BoardPresenceSnapshot>("boardPresence", snapshot => observerEvents.Add(snapshot));
991+
await observer.StartAsync();
992+
await observer.InvokeAsync("JoinBoard", board.Id);
993+
await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1);
994+
995+
// Editor joins and starts editing.
996+
// We use 'await using' so the connection is always disposed (even if an
997+
// assertion throws). The abrupt disconnect is simulated by disposing
998+
// without calling LeaveBoard or SetEditingCard(null) first.
999+
await using var editorConn = SignalRTestHelper.CreateBoardsHubConnection(_factory, editor.Token);
1000+
editorConn.On<BoardPresenceSnapshot>("boardPresence", _ => { });
1001+
await editorConn.StartAsync();
1002+
await editorConn.InvokeAsync("JoinBoard", board.Id);
1003+
await SignalRTestHelper.WaitForEventsAsync(observerEvents, 2); // join event
1004+
1005+
observerEvents.Clear();
1006+
await editorConn.InvokeAsync("SetEditingCard", board.Id, card!.Id);
1007+
1008+
// Wait for editing presence update
1009+
var editingEvents = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1);
1010+
var editorMember = editingEvents.Last().Members
1011+
.FirstOrDefault(m => m.UserId == editor.UserId);
1012+
editorMember.Should().NotBeNull("editor should be visible in presence");
1013+
editorMember!.EditingCardId.Should().Be(card.Id,
1014+
"editor should show as editing the card");
1015+
1016+
// Abrupt disconnect (no LeaveBoard, no SetEditingCard(null))
1017+
observerEvents.Clear();
1018+
await editorConn.DisposeAsync();
1019+
1020+
// Owner should receive a snapshot without the editor
1021+
var afterDisconnect = await SignalRTestHelper.WaitForEventsAsync(observerEvents, 1,
1022+
TimeSpan.FromSeconds(5));
1023+
afterDisconnect.Last().Members.Should().NotContain(m => m.UserId == editor.UserId,
1024+
"disconnected editor should be removed from presence");
1025+
afterDisconnect.Last().Members.Should().ContainSingle(m => m.UserId == owner.UserId,
1026+
"only the owner should remain in presence after editor disconnects");
1027+
}
1028+
1029+
// ── Concurrent Webhook Delivery Creation ────────────────────────────────
1030+
1031+
/// <summary>
1032+
/// Scenario 16: Concurrent board mutations should each create webhook deliveries.
1033+
/// Multiple card operations fire concurrently on a board with an active
1034+
/// webhook subscription. Each mutation should produce its own delivery
1035+
/// record without duplicates or lost events.
1036+
/// </summary>
1037+
[Fact]
1038+
public async Task WebhookDelivery_ConcurrentBoardMutations_EachCreatesDeliveryRecord()
1039+
{
1040+
const int mutationCount = 5;
1041+
1042+
using var client = _factory.CreateClient();
1043+
await ApiTestHarness.AuthenticateAsync(client, "race-webhook-delivery");
1044+
var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-board");
1045+
1046+
var colResp = await client.PostAsJsonAsync(
1047+
$"/api/boards/{board.Id}/columns",
1048+
new CreateColumnDto(board.Id, "Backlog", null, null));
1049+
colResp.StatusCode.Should().Be(HttpStatusCode.Created);
1050+
var col = await colResp.Content.ReadFromJsonAsync<ColumnDto>();
1051+
1052+
// Create a webhook subscription on this board.
1053+
// NOTE: the endpoint URL is external-looking; delivery will fail at send time
1054+
// (no real server) but the delivery RECORD should be created.
1055+
var webhookResp = await client.PostAsJsonAsync(
1056+
$"/api/boards/{board.Id}/webhooks",
1057+
new CreateOutboundWebhookSubscriptionDto(
1058+
"https://example.com/webhook-receiver",
1059+
new List<string> { "card.*" }));
1060+
webhookResp.StatusCode.Should().Be(HttpStatusCode.Created);
1061+
var webhookSub = await webhookResp.Content.ReadFromJsonAsync<OutboundWebhookSubscriptionSecretDto>();
1062+
webhookSub.Should().NotBeNull("webhook subscription should have been created");
1063+
1064+
// Create multiple cards concurrently — each should trigger a webhook delivery.
1065+
// Use Barrier for true synchronization instead of SemaphoreSlim.
1066+
using var barrier = new Barrier(mutationCount + 1);
1067+
var statusCodes = new ConcurrentBag<HttpStatusCode>();
1068+
1069+
var mutationTasks = Enumerable.Range(0, mutationCount).Select(async i =>
1070+
{
1071+
using var raceClient = _factory.CreateClient();
1072+
raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization;
1073+
barrier.SignalAndWait(TimeSpan.FromSeconds(10));
1074+
var resp = await raceClient.PostAsJsonAsync(
1075+
$"/api/boards/{board.Id}/cards",
1076+
new CreateCardDto(board.Id, col!.Id, $"Webhook card {i}", null, null, null));
1077+
statusCodes.Add(resp.StatusCode);
1078+
}).ToArray();
1079+
1080+
barrier.SignalAndWait(TimeSpan.FromSeconds(10));
1081+
await Task.WhenAll(mutationTasks);
1082+
1083+
// All card creations should succeed
1084+
statusCodes.Should().AllSatisfy(s =>
1085+
s.Should().Be(HttpStatusCode.Created),
1086+
"all concurrent card creations should succeed");
1087+
1088+
// Verify all cards were created (no duplicates, no losses)
1089+
var cardsResp = await client.GetAsync($"/api/boards/{board.Id}/cards");
1090+
cardsResp.StatusCode.Should().Be(HttpStatusCode.OK);
1091+
var cards = await cardsResp.Content.ReadFromJsonAsync<List<CardDto>>();
1092+
var webhookCards = cards!.Where(c => c.Title.StartsWith("Webhook card ")).ToList();
1093+
webhookCards.Should().HaveCount(mutationCount,
1094+
"each concurrent mutation should create exactly one card (no duplicates or losses)");
1095+
1096+
// Verify all card titles are unique (no duplicate processing)
1097+
webhookCards.Select(c => c.Title).Distinct().Should().HaveCount(mutationCount,
1098+
"each card title should be unique, proving no duplicate processing");
1099+
1100+
// Verify that webhook delivery records were actually created in the database.
1101+
// Poll with a short timeout because the notifier enqueues deliveries
1102+
// asynchronously after the HTTP response returns.
1103+
using var scope = _factory.Services.CreateScope();
1104+
var deliveryRepo = scope.ServiceProvider.GetRequiredService<IOutboundWebhookDeliveryRepository>();
1105+
1106+
var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10);
1107+
IReadOnlyList<OutboundWebhookDelivery> deliveries = [];
1108+
while (DateTimeOffset.UtcNow < deadline)
1109+
{
1110+
deliveries = await deliveryRepo.GetBySubscriptionAsync(
1111+
webhookSub!.Subscription.Id, limit: mutationCount + 5);
1112+
if (deliveries.Count >= mutationCount)
1113+
{
1114+
break;
1115+
}
1116+
1117+
await Task.Delay(100);
1118+
}
1119+
1120+
deliveries.Should().HaveCount(mutationCount,
1121+
$"each of the {mutationCount} card mutations should create exactly one webhook delivery record");
1122+
deliveries.Select(d => d.Id).Distinct().Should().HaveCount(deliveries.Count,
1123+
"each delivery record should have a unique ID");
1124+
}
1125+
1126+
/// <summary>
1127+
/// Scenario 17: Concurrent webhook subscription creation on the same board.
1128+
/// Multiple webhook subscriptions created simultaneously should all succeed
1129+
/// with distinct IDs and secrets.
1130+
/// </summary>
1131+
[Fact]
1132+
public async Task WebhookSubscription_ConcurrentCreation_AllSucceedWithDistinctIds()
1133+
{
1134+
const int subscriptionCount = 3;
1135+
1136+
using var client = _factory.CreateClient();
1137+
await ApiTestHarness.AuthenticateAsync(client, "race-webhook-sub");
1138+
var board = await ApiTestHarness.CreateBoardAsync(client, "race-webhook-sub-board");
1139+
1140+
// Use Barrier for true synchronization instead of SemaphoreSlim
1141+
using var barrier = new Barrier(subscriptionCount + 1);
1142+
var results = new ConcurrentBag<(HttpStatusCode Status, OutboundWebhookSubscriptionSecretDto? Sub)>();
1143+
1144+
var tasks = Enumerable.Range(0, subscriptionCount).Select(async i =>
1145+
{
1146+
using var raceClient = _factory.CreateClient();
1147+
raceClient.DefaultRequestHeaders.Authorization = client.DefaultRequestHeaders.Authorization;
1148+
barrier.SignalAndWait(TimeSpan.FromSeconds(10));
1149+
var resp = await raceClient.PostAsJsonAsync(
1150+
$"/api/boards/{board.Id}/webhooks",
1151+
new CreateOutboundWebhookSubscriptionDto(
1152+
$"https://example.com/webhook-{i}",
1153+
new List<string> { "card.*" }));
1154+
var sub = resp.StatusCode == HttpStatusCode.Created
1155+
? await resp.Content.ReadFromJsonAsync<OutboundWebhookSubscriptionSecretDto>()
1156+
: null;
1157+
results.Add((resp.StatusCode, sub));
1158+
}).ToArray();
1159+
1160+
barrier.SignalAndWait(TimeSpan.FromSeconds(10));
1161+
await Task.WhenAll(tasks);
1162+
1163+
// All should succeed
1164+
results.Select(r => r.Status).Should().AllSatisfy(s =>
1165+
s.Should().Be(HttpStatusCode.Created),
1166+
"all concurrent webhook subscription creations should succeed");
1167+
1168+
// IDs should be distinct
1169+
var ids = results.Where(r => r.Sub != null).Select(r => r.Sub!.Subscription.Id).ToList();
1170+
ids.Distinct().Should().HaveCount(subscriptionCount,
1171+
"each subscription should have a unique ID");
1172+
1173+
// Signing secrets should be distinct
1174+
var secrets = results.Where(r => r.Sub != null).Select(r => r.Sub!.SigningSecret).ToList();
1175+
secrets.Distinct().Should().HaveCount(subscriptionCount,
1176+
"each subscription should have a unique signing secret");
1177+
1178+
// Verify via list endpoint: parse response and verify distinct IDs and count
1179+
var listResp = await client.GetAsync($"/api/boards/{board.Id}/webhooks");
1180+
listResp.StatusCode.Should().Be(HttpStatusCode.OK);
1181+
var listedSubs = await listResp.Content.ReadFromJsonAsync<List<OutboundWebhookSubscriptionDto>>()
1182+
?? throw new InvalidOperationException("list endpoint returned null subscription data");
1183+
listedSubs.Should().HaveCountGreaterThanOrEqualTo(subscriptionCount,
1184+
$"list endpoint should return at least {subscriptionCount} subscriptions");
1185+
listedSubs.Select(s => s.Id).Distinct().Should().HaveCountGreaterThanOrEqualTo(subscriptionCount,
1186+
"listed subscriptions should have distinct IDs");
1187+
1188+
// Cross-check: all IDs from creation should appear in the list
1189+
foreach (var createdId in ids)
1190+
{
1191+
listedSubs.Should().Contain(s => s.Id == createdId,
1192+
$"subscription {createdId} created concurrently should appear in the list endpoint");
1193+
}
1194+
}
8241195
}

0 commit comments

Comments
 (0)