diff --git a/backend/tests/Taskdeck.Api.Tests/DatabaseProviderCompatibilityTests.cs b/backend/tests/Taskdeck.Api.Tests/DatabaseProviderCompatibilityTests.cs new file mode 100644 index 00000000..3159eb05 --- /dev/null +++ b/backend/tests/Taskdeck.Api.Tests/DatabaseProviderCompatibilityTests.cs @@ -0,0 +1,824 @@ +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Taskdeck.Application.Interfaces; +using Taskdeck.Domain.Entities; +using Taskdeck.Domain.Enums; +using Taskdeck.Infrastructure.Persistence; +using Xunit; + +namespace Taskdeck.Api.Tests; + +/// +/// Provider-compatibility test harness validating that critical persistence +/// operations produce consistent results across database providers. +/// +/// Currently all tests run against SQLite only (the CI and local default). +/// PostgreSQL execution is a future follow-up once the API/runtime wiring +/// supports UseNpgsql() and the test factory can select providers explicitly. +/// +/// Covers: CRUD on core entities (Board, Card, Column, Proposal), query +/// patterns used in application services, date/time round-trip fidelity, +/// string collation behavior, GUID storage, nullable field handling, and +/// batch-write safety. +/// +/// Related: ADR-0023, issue #84 (PLAT-01). +/// +public class DatabaseProviderCompatibilityTests : IClassFixture +{ + private readonly TestWebApplicationFactory _factory; + + public DatabaseProviderCompatibilityTests(TestWebApplicationFactory factory) + { + _factory = factory; + } + + // ─── CRUD: Board ──────────────────────────────────────────────── + + [Fact] + public async Task Board_Create_Read_Update_Delete_RoundTrip() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var repo = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-board-crud", "compat-board@example.com", "hash"); + db.Users.Add(user); + await db.SaveChangesAsync(); + + // Create + var board = new Board("Compat Test Board", "A description", user.Id); + var created = await repo.AddAsync(board); + await db.SaveChangesAsync(); + + created.Id.Should().NotBe(Guid.Empty); + created.Name.Should().Be("Compat Test Board"); + + // Read + var fetched = await repo.GetByIdAsync(board.Id); + fetched.Should().NotBeNull(); + fetched!.Name.Should().Be("Compat Test Board"); + fetched.Description.Should().Be("A description"); + fetched.OwnerId.Should().Be(user.Id); + + // Update + fetched.Update(name: "Updated Board Name"); + await repo.UpdateAsync(fetched); + await db.SaveChangesAsync(); + + var updated = await repo.GetByIdAsync(board.Id); + updated!.Name.Should().Be("Updated Board Name"); + + // Delete + await repo.DeleteAsync(updated); + await db.SaveChangesAsync(); + + var deleted = await repo.GetByIdAsync(board.Id); + deleted.Should().BeNull(); + } + + // ─── CRUD: Card ───────────────────────────────────────────────── + + [Fact] + public async Task Card_Create_Read_Update_Delete_RoundTrip() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var cardRepo = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-card-crud", "compat-card@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Card CRUD Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Todo", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + // Create + var card = new Card(board.Id, column.Id, "Test Card", "Card description", position: 0); + await cardRepo.AddAsync(card); + await db.SaveChangesAsync(); + + // Read + var fetched = await cardRepo.GetByIdAsync(card.Id); + fetched.Should().NotBeNull(); + fetched!.Title.Should().Be("Test Card"); + fetched.Description.Should().Be("Card description"); + fetched.BoardId.Should().Be(board.Id); + fetched.ColumnId.Should().Be(column.Id); + + // Update + fetched.Update(title: "Updated Card Title", description: "Updated description"); + await cardRepo.UpdateAsync(fetched); + await db.SaveChangesAsync(); + + var updated = await cardRepo.GetByIdAsync(card.Id); + updated!.Title.Should().Be("Updated Card Title"); + updated.Description.Should().Be("Updated description"); + + // Delete + await cardRepo.DeleteAsync(updated); + await db.SaveChangesAsync(); + + var deleted = await cardRepo.GetByIdAsync(card.Id); + deleted.Should().BeNull(); + } + + // ─── CRUD: AutomationProposal ─────────────────────────────────── + + [Fact] + public async Task Proposal_Create_Read_StatusTransition_RoundTrip() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var proposalRepo = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-proposal-crud", "compat-proposal@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Proposal Board", ownerId: user.Id); + db.Boards.Add(board); + await db.SaveChangesAsync(); + + // Create + var proposal = new AutomationProposal( + ProposalSourceType.Manual, + user.Id, + "Test proposal summary", + RiskLevel.Low, + Guid.NewGuid().ToString(), + boardId: board.Id); + + await proposalRepo.AddAsync(proposal); + await db.SaveChangesAsync(); + + // Read + var fetched = await proposalRepo.GetByIdAsync(proposal.Id); + fetched.Should().NotBeNull(); + fetched!.Summary.Should().Be("Test proposal summary"); + fetched.Status.Should().Be(ProposalStatus.PendingReview); + fetched.RiskLevel.Should().Be(RiskLevel.Low); + fetched.BoardId.Should().Be(board.Id); + fetched.RequestedByUserId.Should().Be(user.Id); + + // Status transition: Approve → Apply + fetched.Approve(user.Id); + await proposalRepo.UpdateAsync(fetched); + await db.SaveChangesAsync(); + + var approved = await proposalRepo.GetByIdAsync(proposal.Id); + approved!.Status.Should().Be(ProposalStatus.Approved); + approved.DecidedByUserId.Should().Be(user.Id); + approved.DecidedAt.Should().NotBeNull(); + + approved.MarkAsApplied(); + await proposalRepo.UpdateAsync(approved); + await db.SaveChangesAsync(); + + var applied = await proposalRepo.GetByIdAsync(proposal.Id); + applied!.Status.Should().Be(ProposalStatus.Applied); + applied.AppliedAt.Should().NotBeNull(); + } + + // ─── DateTimeOffset round-trip fidelity ───────────────────────── + + [Fact] + public async Task DateTimeOffset_RoundTrip_PreservesUtcPrecision() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-dt-user", "compat-dt@example.com", "hash"); + db.Users.Add(user); + + var board = new Board("DateTime Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + // Use a specific DueDate with subsecond precision + var specificDate = new DateTimeOffset(2026, 6, 15, 14, 30, 45, 123, TimeSpan.Zero); + var card = new Card(board.Id, column.Id, "DateTime Card", dueDate: specificDate, position: 0); + db.Cards.Add(card); + await db.SaveChangesAsync(); + + // Clear tracker and re-fetch + db.ChangeTracker.Clear(); + var fetched = await db.Cards.FirstAsync(c => c.Id == card.Id); + + fetched.DueDate.Should().NotBeNull(); + // Whole-second precision is the reliable cross-provider minimum. + // SQLite stores as text; PostgreSQL as timestamptz. Both preserve seconds. + fetched.DueDate!.Value.Year.Should().Be(2026); + fetched.DueDate.Value.Month.Should().Be(6); + fetched.DueDate.Value.Day.Should().Be(15); + fetched.DueDate.Value.Hour.Should().Be(14); + fetched.DueDate.Value.Minute.Should().Be(30); + fetched.DueDate.Value.Second.Should().Be(45); + } + + [Fact] + public async Task CreatedAt_UpdatedAt_ArePreservedOnRoundTrip() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-timestamps", "compat-ts@example.com", "hash"); + var createdBefore = DateTimeOffset.UtcNow; + db.Users.Add(user); + + var board = new Board("Timestamps Board", ownerId: user.Id); + db.Boards.Add(board); + await db.SaveChangesAsync(); + var createdAfter = DateTimeOffset.UtcNow; + + db.ChangeTracker.Clear(); + var fetched = await db.Boards.FirstAsync(b => b.Id == board.Id); + + // CreatedAt should be between our before/after markers + fetched.CreatedAt.Should().BeOnOrAfter(createdBefore.AddSeconds(-1)); + fetched.CreatedAt.Should().BeOnOrBefore(createdAfter.AddSeconds(1)); + + // UpdatedAt should equal CreatedAt initially + fetched.UpdatedAt.Should().BeCloseTo(fetched.CreatedAt, TimeSpan.FromSeconds(2)); + + // Update and verify UpdatedAt changes + fetched.Update(name: "Updated Timestamps Board"); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + var updated = await db.Boards.FirstAsync(b => b.Id == board.Id); + updated.UpdatedAt.Should().BeOnOrAfter(fetched.CreatedAt); + } + + // ─── GUID storage and retrieval ───────────────────────────────── + + [Fact] + public async Task Guid_PreservesExactValue_AcrossRoundTrip() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-guid-test", "compat-guid@example.com", "hash"); + db.Users.Add(user); + + var board = new Board("GUID Board", ownerId: user.Id); + var originalBoardId = board.Id; + var originalUserId = user.Id; + db.Boards.Add(board); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + var fetchedBoard = await db.Boards.FirstAsync(b => b.Id == originalBoardId); + var fetchedUser = await db.Users.FirstAsync(u => u.Id == originalUserId); + + fetchedBoard.Id.Should().Be(originalBoardId); + fetchedBoard.OwnerId.Should().Be(originalUserId); + fetchedUser.Id.Should().Be(originalUserId); + } + + [Fact] + public async Task Guid_ForeignKey_JoinQuery_WorksCorrectly() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-guid-fk", "compat-fk@example.com", "hash"); + db.Users.Add(user); + var board = new Board("FK Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "FK Column", 0); + db.Columns.Add(column); + var card = new Card(board.Id, column.Id, "FK Card", position: 0); + db.Cards.Add(card); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // Join query: cards with their columns + var result = await db.Cards + .Where(c => c.BoardId == board.Id) + .Join(db.Columns, c => c.ColumnId, col => col.Id, (c, col) => new { Card = c, Column = col }) + .FirstOrDefaultAsync(); + + result.Should().NotBeNull(); + result!.Card.Id.Should().Be(card.Id); + result.Column.Id.Should().Be(column.Id); + } + + // ─── String collation behavior ────────────────────────────────── + + [Fact] + public async Task String_CaseSensitiveComparison_IsConsistent() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-collation", "compat-collation@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Collation Test Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + var cardUpper = new Card(board.Id, column.Id, "IMPORTANT TASK", position: 0); + var cardLower = new Card(board.Id, column.Id, "important task", position: 1); + var cardMixed = new Card(board.Id, column.Id, "Important Task", position: 2); + db.Cards.AddRange(cardUpper, cardLower, cardMixed); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // Exact equality (==) is case-sensitive on both SQLite and PostgreSQL. + // This confirms both providers agree on the behavior. + var exactMatch = await db.Cards + .Where(c => c.BoardId == board.Id && c.Title == "IMPORTANT TASK") + .ToListAsync(); + + exactMatch.Should().ContainSingle(); + exactMatch[0].Id.Should().Be(cardUpper.Id); + } + + [Fact] + public async Task String_ContainsQuery_BehaviorIsConsistent() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-contains", "compat-contains@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Contains Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + var card1 = new Card(board.Id, column.Id, "Fix login bug", position: 0); + var card2 = new Card(board.Id, column.Id, "Add LOGIN feature", position: 1); + var card3 = new Card(board.Id, column.Id, "Update docs", position: 2); + db.Cards.AddRange(card1, card2, card3); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // EF Core translates Contains() to case-sensitive matching on both + // SQLite (via instr() or LIKE with PRAGMA case_sensitive_like) and + // PostgreSQL (via LIKE '%value%'). Both providers match only "Fix login bug" + // for Contains("login") — "Add LOGIN feature" does NOT match. + // For case-insensitive search on PostgreSQL, use EF.Functions.ILike(). + // For case-insensitive search on SQLite, raw LIKE (without EF Core) is + // case-insensitive for ASCII, but EF Core's translation is case-sensitive. + var containsLogin = await db.Cards + .Where(c => c.BoardId == board.Id && c.Title.Contains("login")) + .ToListAsync(); + + // EF Core's Contains() is case-sensitive on both SQLite and PostgreSQL + containsLogin.Count.Should().Be(1, + "EF Core Contains() is case-sensitive — only exact-case 'login' matches"); + } + + // ─── Nullable field handling ──────────────────────────────────── + + [Fact] + public async Task Nullable_Fields_HandleNullCorrectly() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-nullable", "compat-nullable@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Nullable Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + // Card with null DueDate + var cardNullDue = new Card(board.Id, column.Id, "No due date", position: 0); + // Card with a DueDate + var cardWithDue = new Card(board.Id, column.Id, "Has due date", + dueDate: DateTimeOffset.UtcNow.AddDays(7), position: 1); + db.Cards.AddRange(cardNullDue, cardWithDue); + + // Board with null Description + var boardNullDesc = new Board("No Desc Board", ownerId: user.Id); + db.Boards.Add(boardNullDesc); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + var fetchedNullDue = await db.Cards.FirstAsync(c => c.Id == cardNullDue.Id); + fetchedNullDue.DueDate.Should().BeNull(); + + var fetchedWithDue = await db.Cards.FirstAsync(c => c.Id == cardWithDue.Id); + fetchedWithDue.DueDate.Should().NotBeNull(); + + var fetchedNullDesc = await db.Boards.FirstAsync(b => b.Id == boardNullDesc.Id); + fetchedNullDesc.Description.Should().BeNull(); + + // Query filtering on null + var cardsWithoutDueDate = await db.Cards + .Where(c => c.BoardId == board.Id && c.DueDate == null) + .ToListAsync(); + cardsWithoutDueDate.Should().Contain(c => c.Id == cardNullDue.Id); + cardsWithoutDueDate.Should().NotContain(c => c.Id == cardWithDue.Id); + } + + // ─── Query patterns used in application services ──────────────── + + [Fact] + public async Task BoardWithDetails_IncludesColumnsAndCards() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var boardRepo = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-details", "compat-details@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Details Board", ownerId: user.Id); + db.Boards.Add(board); + var col1 = new Column(board.Id, "Todo", 0); + var col2 = new Column(board.Id, "Done", 1); + db.Columns.AddRange(col1, col2); + await db.SaveChangesAsync(); + + var card1 = new Card(board.Id, col1.Id, "Card in Todo", position: 0); + var card2 = new Card(board.Id, col2.Id, "Card in Done", position: 0); + db.Cards.AddRange(card1, card2); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + var detailed = await boardRepo.GetByIdWithDetailsAsync(board.Id); + detailed.Should().NotBeNull(); + detailed!.Columns.Should().HaveCount(2); + detailed.Cards.Should().HaveCount(2); + } + + [Fact] + public async Task ReadableBoards_FiltersByOwnerAndAccess() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var boardRepo = scope.ServiceProvider.GetRequiredService(); + + var owner = new User("compat-readable-owner", "compat-ro@example.com", "hash"); + var collaborator = new User("compat-readable-collab", "compat-rc@example.com", "hash"); + var outsider = new User("compat-readable-outsider", "compat-rout@example.com", "hash"); + db.Users.AddRange(owner, collaborator, outsider); + + var ownedBoard = new Board("Owned Board", ownerId: owner.Id); + var sharedBoard = new Board("Shared Board", ownerId: owner.Id); + db.Boards.AddRange(ownedBoard, sharedBoard); + await db.SaveChangesAsync(); + + var access = new BoardAccess(sharedBoard.Id, collaborator.Id, UserRole.Editor, owner.Id); + db.BoardAccesses.Add(access); + await db.SaveChangesAsync(); + + // Owner sees both boards + var ownerBoards = (await boardRepo.GetReadableByUserIdAsync(owner.Id, includeArchived: false)).ToList(); + ownerBoards.Should().Contain(b => b.Id == ownedBoard.Id); + ownerBoards.Should().Contain(b => b.Id == sharedBoard.Id); + + // Collaborator sees only shared board + var collabBoards = (await boardRepo.GetReadableByUserIdAsync(collaborator.Id, includeArchived: false)).ToList(); + collabBoards.Should().Contain(b => b.Id == sharedBoard.Id); + collabBoards.Should().NotContain(b => b.Id == ownedBoard.Id); + + // Outsider sees neither + var outsiderBoards = (await boardRepo.GetReadableByUserIdAsync(outsider.Id, includeArchived: false)).ToList(); + outsiderBoards.Should().NotContain(b => b.Id == ownedBoard.Id); + outsiderBoards.Should().NotContain(b => b.Id == sharedBoard.Id); + } + + // ─── Ordering and pagination queries ──────────────────────────── + + [Fact] + public async Task OrderBy_IntegerColumn_ReturnsConsistentOrder() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-order", "compat-order@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Order Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + for (int i = 0; i < 5; i++) + { + var card = new Card(board.Id, column.Id, $"Order Card {i}", position: 4 - i); + db.Cards.Add(card); + } + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // Order by Position ascending + var orderedAsc = await db.Cards + .Where(c => c.BoardId == board.Id) + .OrderBy(c => c.Position) + .ToListAsync(); + + orderedAsc.Should().HaveCount(5); + for (int i = 1; i < orderedAsc.Count; i++) + { + orderedAsc[i].Position.Should().BeGreaterThanOrEqualTo(orderedAsc[i - 1].Position); + } + + // Order by Position descending + var orderedDesc = await db.Cards + .Where(c => c.BoardId == board.Id) + .OrderByDescending(c => c.Position) + .ToListAsync(); + + orderedDesc.Should().HaveCount(5); + for (int i = 1; i < orderedDesc.Count; i++) + { + orderedDesc[i].Position.Should().BeLessThanOrEqualTo(orderedDesc[i - 1].Position); + } + } + + /// + /// Documents that SQLite does not support ORDER BY on DateTimeOffset columns. + /// PostgreSQL (timestamptz) handles this natively. Application code must use + /// materialize-then-sort or cast to string for SQLite compatibility. + /// This is a known provider difference — see ADR-0023. + /// + [Fact] + public async Task DateTimeOffset_OrderBy_RequiresClientSideForSqlite() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-dto-order", "compat-dto-order@example.com", "hash"); + db.Users.Add(user); + var board = new Board("DTO Order Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + for (int i = 0; i < 3; i++) + { + db.Cards.Add(new Card(board.Id, column.Id, $"DT Card {i}", position: i)); + } + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // SQLite throws NotSupportedException for DateTimeOffset ORDER BY. + // The workaround is to materialize first, then sort client-side. + var cards = await db.Cards + .Where(c => c.BoardId == board.Id) + .ToListAsync(); + + var sortedClientSide = cards.OrderBy(c => c.CreatedAt).ToList(); + sortedClientSide.Should().HaveCount(3); + for (int i = 1; i < sortedClientSide.Count; i++) + { + sortedClientSide[i].CreatedAt.Should().BeOnOrAfter(sortedClientSide[i - 1].CreatedAt); + } + } + + [Fact] + public async Task Skip_Take_PaginationQuery_WorksCorrectly() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-page", "compat-page@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Pagination Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + for (int i = 0; i < 10; i++) + { + db.Cards.Add(new Card(board.Id, column.Id, $"Page Card {i:D2}", position: i)); + } + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // Page 1: skip 0, take 3 + var page1 = await db.Cards + .Where(c => c.BoardId == board.Id) + .OrderBy(c => c.Position) + .Skip(0).Take(3) + .ToListAsync(); + page1.Should().HaveCount(3); + + // Page 2: skip 3, take 3 + var page2 = await db.Cards + .Where(c => c.BoardId == board.Id) + .OrderBy(c => c.Position) + .Skip(3).Take(3) + .ToListAsync(); + page2.Should().HaveCount(3); + + // No overlap between pages + page1.Select(c => c.Id).Should().NotIntersectWith(page2.Select(c => c.Id)); + + // Total count + var total = await db.Cards.CountAsync(c => c.BoardId == board.Id); + total.Should().Be(10); + } + + // ─── Enum storage and filtering ───────────────────────────────── + + [Fact] + public async Task Enum_StorageAndFiltering_WorksAcrossProviders() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-enum", "compat-enum@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Enum Board", ownerId: user.Id); + db.Boards.Add(board); + await db.SaveChangesAsync(); + + // Create proposals with different statuses and risk levels + var lowProposal = new AutomationProposal( + ProposalSourceType.Manual, user.Id, "Low risk", + RiskLevel.Low, Guid.NewGuid().ToString(), board.Id); + var highProposal = new AutomationProposal( + ProposalSourceType.Chat, user.Id, "High risk", + RiskLevel.High, Guid.NewGuid().ToString(), board.Id); + + db.AutomationProposals.AddRange(lowProposal, highProposal); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // Filter by enum value + var lowRisk = await db.AutomationProposals + .Where(p => p.BoardId == board.Id && p.RiskLevel == RiskLevel.Low) + .ToListAsync(); + lowRisk.Should().ContainSingle(); + lowRisk[0].Id.Should().Be(lowProposal.Id); + + // Filter by source type + var chatProposals = await db.AutomationProposals + .Where(p => p.BoardId == board.Id && p.SourceType == ProposalSourceType.Chat) + .ToListAsync(); + chatProposals.Should().ContainSingle(); + chatProposals[0].Id.Should().Be(highProposal.Id); + } + + // ─── Aggregate queries ────────────────────────────────────────── + + [Fact] + public async Task Aggregate_CountAndGroupBy_WorkCorrectly() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-agg", "compat-agg@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Aggregate Board", ownerId: user.Id); + db.Boards.Add(board); + var col1 = new Column(board.Id, "Todo", 0); + var col2 = new Column(board.Id, "Done", 1); + db.Columns.AddRange(col1, col2); + await db.SaveChangesAsync(); + + db.Cards.AddRange( + new Card(board.Id, col1.Id, "Agg Card 1", position: 0), + new Card(board.Id, col1.Id, "Agg Card 2", position: 1), + new Card(board.Id, col1.Id, "Agg Card 3", position: 2), + new Card(board.Id, col2.Id, "Agg Card 4", position: 0) + ); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // Count per column + var countByColumn = await db.Cards + .Where(c => c.BoardId == board.Id) + .GroupBy(c => c.ColumnId) + .Select(g => new { ColumnId = g.Key, Count = g.Count() }) + .ToListAsync(); + + countByColumn.Should().HaveCount(2); + countByColumn.First(g => g.ColumnId == col1.Id).Count.Should().Be(3); + countByColumn.First(g => g.ColumnId == col2.Id).Count.Should().Be(1); + + // Total count + var total = await db.Cards.CountAsync(c => c.BoardId == board.Id); + total.Should().Be(4); + } + + // ─── Boolean field filtering ──────────────────────────────────── + + [Fact] + public async Task Boolean_FilteringWorks_AcrossProviders() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-bool", "compat-bool@example.com", "hash"); + db.Users.Add(user); + + var activeBoard = new Board("Active", ownerId: user.Id); + var archivedBoard = new Board("Archived", ownerId: user.Id); + archivedBoard.Archive(); + db.Boards.AddRange(activeBoard, archivedBoard); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + var active = await db.Boards + .Where(b => b.OwnerId == user.Id && !b.IsArchived) + .ToListAsync(); + active.Should().ContainSingle(); + active[0].Id.Should().Be(activeBoard.Id); + + var archived = await db.Boards + .Where(b => b.OwnerId == user.Id && b.IsArchived) + .ToListAsync(); + archived.Should().ContainSingle(); + archived[0].Id.Should().Be(archivedBoard.Id); + } + + // ─── Batch writes (basic safety) ─────────────────────────────── + + /// + /// Validates that a batch of inserts in a single SaveChangesAsync call + /// persists all rows correctly. This is NOT a true concurrency test — + /// it uses a single DbContext on a single thread. Real multi-context + /// concurrent write testing should be added when PostgreSQL support + /// is available (SQLite uses database-level write locking). + /// + [Fact] + public async Task BatchInsert_DoesNotLoseData() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-concurrent", "compat-concurrent@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Concurrent Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + // Insert 20 cards in a single batch (not truly concurrent) + var cardIds = new List(); + for (int i = 0; i < 20; i++) + { + var card = new Card(board.Id, column.Id, $"Concurrent Card {i}", position: i); + cardIds.Add(card.Id); + db.Cards.Add(card); + } + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + // All 20 cards should be persisted + var count = await db.Cards.CountAsync(c => c.BoardId == board.Id); + count.Should().Be(20); + + // All IDs should be retrievable + var retrievedIds = await db.Cards + .Where(c => c.BoardId == board.Id) + .Select(c => c.Id) + .ToListAsync(); + retrievedIds.Should().BeEquivalentTo(cardIds); + } + + // ─── Unicode string handling ──────────────────────────────────── + + [Fact] + public async Task Unicode_Strings_PreservedOnRoundTrip() + { + using var scope = _factory.Services.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + + var user = new User("compat-unicode", "compat-unicode@example.com", "hash"); + db.Users.Add(user); + var board = new Board("Unicode Board", ownerId: user.Id); + db.Boards.Add(board); + var column = new Column(board.Id, "Col", 0); + db.Columns.Add(column); + await db.SaveChangesAsync(); + + var unicodeTitle = "Fix bug in \u65E5\u672C\u8A9E module \u2014 \u00FC\u00F6\u00E4 chars & emojis"; + var card = new Card(board.Id, column.Id, unicodeTitle, position: 0); + db.Cards.Add(card); + await db.SaveChangesAsync(); + + db.ChangeTracker.Clear(); + + var fetched = await db.Cards.FirstAsync(c => c.Id == card.Id); + fetched.Title.Should().Be(unicodeTitle); + } +} diff --git a/docs/IMPLEMENTATION_MASTERPLAN.md b/docs/IMPLEMENTATION_MASTERPLAN.md index 14374edb..9154f40e 100644 --- a/docs/IMPLEMENTATION_MASTERPLAN.md +++ b/docs/IMPLEMENTATION_MASTERPLAN.md @@ -620,25 +620,30 @@ Delivered in the latest cycle: - resilience/degraded-mode tests (`#720`/`#778`): 34 tests (18 backend + 16 frontend); adversarial review fixed CI blocker (unused import), double-invocation anti-pattern, and timing race - E2E error state expansion (`#712`/`#772`): 25 Playwright scenarios across 3 spec files using `page.route()` interception; adversarial review fixed CI blocker (unused import), route glob, and 3 vacuous assertions - TST-32–TST-57 wave: 23 of 25 issues now delivered (added `#723`/`#769` and `#725`/`#765` from parallel wave); remaining open: `#705`, `#717`; frontend suite ~1734 passing -130. Ephemeral integration databases via Testcontainers (`#91`, 2026-04-09): - - new `Taskdeck.Integration.Tests` project with `Testcontainers.PostgreSql` (4.11.0) and `Npgsql.EntityFrameworkCore.PostgreSQL` (8.0.11) - - `PostgresContainerFixture` manages a shared ephemeral PostgreSQL 16 container per xUnit collection; each test method gets its own isolated database via counter-based `CREATE DATABASE` +130. Ephemeral integration databases via Testcontainers (`#91`, 2026-04-09): + - new `Taskdeck.Integration.Tests` project with `Testcontainers.PostgreSql` (4.11.0) and `Npgsql.EntityFrameworkCore.PostgreSQL` (8.0.11) + - `PostgresContainerFixture` manages a shared ephemeral PostgreSQL 16 container per xUnit collection; each test method gets its own isolated database via counter-based `CREATE DATABASE` - schema created via `EnsureCreated()` from the EF Core model (not SQLite migrations) for PostgreSQL provider parity - `PostgresIntegrationTestBase` base class provides `Db` property with `IAsyncLifetime` setup/teardown - 20 integration tests across 7 test classes: Board CRUD (5), Card operations (5), Proposal lifecycle (5), per-test isolation verification (2), parallel execution validation (3) - CI workflow `reusable-container-integration.yml` added to ci-extended lane (label: testing); runs on ubuntu-latest with Docker - documentation at `docs/testing/TESTCONTAINERS_GUIDE.md` -131. SignalR scale-out readiness (`#105`, PLAT-03, 2026-04-09): - - ADR-0023 documents Redis backplane strategy with alternatives analysis (Azure SignalR Service, custom message bus, sticky sessions) - - `Microsoft.AspNetCore.SignalR.StackExchangeRedis` 8.0.25 added with conditional activation: Redis backplane enabled when `SignalR:Redis:ConnectionString` configured, in-memory fallback when absent - - `RedisBackplaneHealthCheck` reports NotConfigured/Healthy/Unhealthy in `/health/ready` endpoint - - `SignalRRegistration` extension replaces bare `AddSignalR()` with configurable builder - - operational runbook at `docs/platform/SIGNALR_SCALEOUT_RUNBOOK.md` covers Docker Compose multi-instance, load balancer WebSocket config, failure scenarios, and rollback - - 14 new tests: configuration detection, logging, health check states, readiness endpoint integration, hub negotiate preservation - -132. Platform expansion wave delivery (PRs `#796`–`#805`, 2026-04-09): +131. SignalR scale-out readiness (`#105`, PLAT-03, 2026-04-09): + - ADR-0023 documents Redis backplane strategy with alternatives analysis (Azure SignalR Service, custom message bus, sticky sessions) + - `Microsoft.AspNetCore.SignalR.StackExchangeRedis` 8.0.25 added with conditional activation: Redis backplane enabled when `SignalR:Redis:ConnectionString` configured, in-memory fallback when absent + - `RedisBackplaneHealthCheck` reports NotConfigured/Healthy/Unhealthy in `/health/ready` endpoint + - `SignalRRegistration` extension replaces bare `AddSignalR()` with configurable builder + - operational runbook at `docs/platform/SIGNALR_SCALEOUT_RUNBOOK.md` covers Docker Compose multi-instance, load balancer WebSocket config, failure scenarios, and rollback + - 14 new tests: configuration detection, logging, health check states, readiness endpoint integration, hub negotiate preservation +132. SQLite-to-PostgreSQL production migration strategy (`#84`, 2026-04-09): + - ADR-0023: recommends PostgreSQL as the production target and documents the alternatives/tradeoffs; runtime provider switching remains follow-up implementation work because `AddInfrastructure()` is still SQLite-only + - migration runbook at `docs/platform/SQLITE_TO_POSTGRES_MIGRATION_RUNBOOK.md`: explicit blocker notes for runtime provider wiring and SQLite-only FTS migration SQL, dependency-ordered export/import, full row-count/FK verification scope, rollback procedure, and least-privilege/security guidance + - 20-test SQLite-backed provider-compatibility baseline (`DatabaseProviderCompatibilityTests`): CRUD on Board/Card/Proposal, DateTimeOffset fidelity, GUID storage and FK joins, string collation, ordering, pagination, enum storage, aggregates, boolean filtering, batch inserts, Unicode; documents SQLite `DateTimeOffset` ORDER BY limitation + - future follow-up: add runtime `UseNpgsql()` support plus a provider-switching test factory before claiming dual-provider execution + +133. Platform expansion wave delivery (PRs `#796`–`#805`, 2026-04-09): - 10 parallel worktree agents delivered platform hardening, testing infrastructure, ops documentation, and PWA readiness with two rounds of adversarial review per PR (22 CRITICAL + 32 HIGH findings caught and resolved) - - **PLAT-01** SQLite-to-PostgreSQL migration strategy (`#84`/`#801`): ADR-0023 (PostgreSQL target), migration runbook, 20 provider compatibility tests; review caught phantom table, 5 missing tables, FTS5 crash + - **PLAT-01** SQLite-to-PostgreSQL migration strategy (`#84`/`#801`): ADR-0023 (PostgreSQL target with runtime follow-up explicitly called out), migration runbook, 20 SQLite-backed provider compatibility baseline tests; review caught provider-switch overstatement, missing verification tables, and FTS5 crash risk - **PLAT-02** Distributed caching (`#85`/`#805`): ADR-0024 (cache-aside), `ICacheService` with Redis/InMemory/NoOp implementations, board list caching, 32 tests; review removed unsafe board-detail cache, fixed permanent Redis disable - **PLAT-03** SignalR scale-out (`#105`/`#803`): ADR-0025 (Redis backplane), conditional `AddTaskdeckSignalR`, health check, runbook, 14 tests; review fixed per-probe connection creation, thread-unsafe fields - **TST-02** Cross-browser E2E matrix (`#87`/`#800`): Firefox/WebKit/mobile projects, tagging strategy, 9 tests, CI workflows, flaky test policy; review fixed CI gate timeout, extracted shared helpers diff --git a/docs/STATUS.md b/docs/STATUS.md index af3ee9c3..3e182d51 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -119,6 +119,8 @@ Current constraints are mostly hardening and consistency: - Mutation testing pilot now delivered (`#90`): Stryker.NET targeting `Taskdeck.Domain` (backend) and Stryker JS targeting `captureStore`/`boardStore` (frontend); non-blocking weekly CI lane (`.github/workflows/mutation-testing.yml`); policy and triage guidance at `docs/testing/MUTATION_TESTING_POLICY.md`; 60% low / 80% high thresholds with 0% break (triage signal, not enforcement gate); scope expansion roadmap covers Application layer and additional frontend stores +- SQLite-to-PostgreSQL production migration strategy delivered (`#84`): ADR-0023 documents PostgreSQL as the production target while explicitly noting the runtime remains SQLite-only today; migration runbook at `docs/platform/SQLITE_TO_POSTGRES_MIGRATION_RUNBOOK.md` now captures the real blockers (missing runtime provider switch, SQLite-only FTS migration SQL) and the full table-verification scope, including `ApiKeys`; 20 SQLite-backed provider-compatibility baseline tests in `DatabaseProviderCompatibilityTests` document the persistence behaviors PostgreSQL support must preserve, including the known SQLite `DateTimeOffset` ORDER BY limitation and batch-insert sanity coverage + Target experience metrics for the capture direction: - capture action to saved artifact should feel under 10 seconds in normal use - capture artifact to reviewed/applicable proposal should be achievable inside a ~60-second loop @@ -260,7 +262,7 @@ Direction guardrails (explicit): Ten parallel worktree agents delivered platform hardening, testing infrastructure, ops documentation, and PWA readiness across 10 PRs with two rounds of adversarial review per PR. All CRITICAL and HIGH findings were resolved. **Architecture & Platform:** -- **PLAT-01 SQLite-to-PostgreSQL migration strategy** (`#84`/`#801`): ADR-0023 recommends PostgreSQL as production target; migration runbook at `docs/platform/SQLITE_TO_POSTGRES_MIGRATION_RUNBOOK.md` with dependency-ordered export/import, FTS5 blocker warning, rollback procedure; 20 provider compatibility tests in `DatabaseProviderCompatibilityTests.cs` covering CRUD, DateTimeOffset, GUID, collation, Unicode; adversarial review caught phantom ApiKeys table, 5 missing tables, FTS5 crash risk +- **PLAT-01 SQLite-to-PostgreSQL migration strategy** (`#84`/`#801`): ADR-0023 recommends PostgreSQL as the production target, but the runtime/provider switch remains follow-up implementation work; migration runbook at `docs/platform/SQLITE_TO_POSTGRES_MIGRATION_RUNBOOK.md` now accurately documents the current blockers, least-privilege provisioning, dependency-ordered export/import, and full verification scope; `DatabaseProviderCompatibilityTests.cs` is a SQLite-backed compatibility baseline covering CRUD, DateTimeOffset, GUID, collation, Unicode, and batch-insert persistence semantics - **PLAT-02 Distributed caching** (`#85`/`#805`): ADR-0024 documents cache-aside pattern; `ICacheService` interface in Application layer; `InMemoryCacheService` (ConcurrentDictionary + sweep timer + 10K cap), `RedisCacheService` (lazy reconnect, safe degradation), `NoOpCacheService`; board list caching with 60s TTL and write-through invalidation; `CacheSettings` config binding; 32 tests; adversarial review removed stale board-detail cache (columns mutated by non-cache-aware services), fixed permanent Redis disable on transient failure, added eviction and timer safety - **PLAT-03 SignalR scale-out** (`#105`/`#803`): ADR-0025 documents Redis backplane strategy; conditional `AddTaskdeckSignalR` extension with `SignalR:Redis:ConnectionString` toggle; `RedisBackplaneHealthCheck` with 30s cache and three-state reporting (NotConfigured/Healthy/Unhealthy); runbook at `docs/platform/SIGNALR_SCALEOUT_RUNBOOK.md`; 14 tests; adversarial review replaced per-probe ConnectionMultiplexer with singleton lazy connection, fixed thread-unsafe cache fields, corrected ADR Degraded/Unhealthy mismatch diff --git a/docs/decisions/ADR-0023-sqlite-to-postgresql-migration-strategy.md b/docs/decisions/ADR-0023-sqlite-to-postgresql-migration-strategy.md new file mode 100644 index 00000000..807f8491 --- /dev/null +++ b/docs/decisions/ADR-0023-sqlite-to-postgresql-migration-strategy.md @@ -0,0 +1,93 @@ +# ADR-0023: SQLite-to-PostgreSQL Production Migration Strategy + +- **Status**: Accepted +- **Date**: 2026-04-09 +- **Deciders**: Project maintainers + +## Context + +Taskdeck currently uses SQLite as its sole persistence provider (ADR-0001 clean architecture, local-first thesis). SQLite is ideal for the current single-user, local-first deployment model — zero-config, file-based, and embedded in the application process. + +However, the platform expansion strategy (ADR-0014, issue #531) targets hosted cloud deployment (v0.2.0) and collaboration features (v0.4.0). These milestones require a production database that supports: + +- Concurrent write access from multiple API instances +- Connection pooling for horizontal scaling +- Row-level locking instead of database-level locking +- Robust backup, point-in-time recovery, and replication +- Managed hosting on all major cloud platforms (AWS RDS, Azure Database, GCP Cloud SQL) + +The project needs a clear provider choice, a migration path, and a compatibility harness to catch provider-specific regressions before they reach production. + +## Decision + +**Adopt PostgreSQL as the target production database provider.** SQLite remains the default for local development, self-contained single-user deployments, and CI test runs. + +The migration strategy is: + +1. **Provider target decision now, runtime switch later**: PostgreSQL is the production target, but the current application runtime still hard-wires `UseSqlite()` in `Taskdeck.Infrastructure.DependencyInjection`. Adding runtime provider selection and Npgsql registration is follow-up implementation work, not something this ADR PR ships. + +2. **SQLite-backed compatibility baseline first**: `DatabaseProviderCompatibilityTests` establishes the persistence behaviors that PostgreSQL support must preserve. Today those tests run against SQLite only. A future follow-up can add a provider-switching test factory and opt-in PostgreSQL execution once the runtime path exists. + +3. **Schema migration remains blocked on follow-up implementation**: EF Core migrations stay the source of truth, but PostgreSQL schema application cannot be treated as ready until SQLite-only migration SQL (notably the FTS5 migration) is wrapped in provider-conditional logic and the application/test infrastructure can actually build `UseNpgsql()` contexts. + +4. **Data migration planning uses row-count and foreign-key integrity verification**: The runbook documents dependency-ordered export/import and verification steps, but it is explicitly preparatory until the provider-switching and PostgreSQL-safe migration work lands. + +## Alternatives Considered + +### SQL Server + +- **Pros**: First-class EF Core support, strong enterprise adoption, Azure-native. +- **Cons**: License cost for production use (Express edition has 10 GB limit), heavier resource footprint than PostgreSQL, less natural fit for the open-source/local-first ethos. Cloud portability is weaker — Azure SQL is easy, but AWS and GCP managed SQL Server options are more expensive and less common. +- **Verdict**: Rejected. The open-source, multi-cloud positioning of PostgreSQL better fits Taskdeck's platform expansion goals. + +### CockroachDB + +- **Pros**: PostgreSQL wire-compatible, built-in distributed SQL, strong horizontal scaling. +- **Cons**: Operational complexity disproportionate to Taskdeck's near-term scale requirements. CockroachDB's serverless tier has cold-start latency. EF Core compatibility is good but not identical to native PostgreSQL (some DDL differences, limited FTS support). The team would be adopting two new technologies (PostgreSQL dialect + CockroachDB operations) simultaneously. +- **Verdict**: Rejected for initial production deployment. Can be revisited if Taskdeck reaches scale requiring distributed SQL, since the PostgreSQL migration path makes CockroachDB a viable future option. + +### MySQL / MariaDB + +- **Pros**: Wide adoption, managed options on all clouds. +- **Cons**: EF Core's MySQL provider (Pomelo) is community-maintained rather than Microsoft-supported. `DateTimeOffset` handling requires workarounds. GUID storage is less ergonomic than PostgreSQL's native `uuid` type. Full-text search capabilities are weaker. +- **Verdict**: Rejected. PostgreSQL offers better EF Core alignment and richer type support. + +### Keep SQLite for all deployments + +- **Pros**: Zero migration effort, proven local-first behavior. +- **Cons**: Database-level write locking makes concurrent multi-user access impractical. No connection pooling. Backup and replication require file-system-level coordination. Not viable for the cloud/collaboration milestones in the platform expansion strategy. +- **Verdict**: Rejected for hosted deployments. SQLite remains the default for local/single-user mode. + +## Consequences + +### Positive + +- PostgreSQL is open-source (PostgreSQL License), eliminating license cost concerns. +- Native `uuid`, `timestamptz`, `jsonb`, and full-text search types align well with the existing domain model (GUIDs, DateTimeOffset, JSON metadata columns, knowledge document FTS). +- EF Core's Npgsql provider is mature, Microsoft-co-maintained, and supports all EF Core features used in the project. +- Managed PostgreSQL is available on AWS (RDS/Aurora), Azure (Flexible Server), and GCP (Cloud SQL) with sub-$20/month entry points. +- The SQLite-backed compatibility harness documents the persistence behaviors that PostgreSQL support must preserve. +- CockroachDB remains a future option due to PostgreSQL wire compatibility. + +### Negative + +- The runtime application projects do not yet reference `Npgsql.EntityFrameworkCore.PostgreSQL` or expose a provider-selection path. +- SQLite-specific constructs (FTS5 virtual tables in `KnowledgeDocuments`) require provider-conditional migration code before PostgreSQL schema creation is possible. +- CI and the API test factory do not yet have a PostgreSQL execution path. +- The current runbook is therefore a preparatory operator document, not a fully executable migration recipe. + +### Neutral + +- No domain-layer or application-layer changes are required — the provider switch is entirely in Infrastructure and DI configuration. +- The existing `IsSqlite()` pattern (used in `AgentRunRepository`) provides a precedent for provider-conditional logic. +- Local development continues to use SQLite unless a developer opts into PostgreSQL. + +## References + +- Issue: #84 (PLAT-01: SQLite-to-production DB migration strategy) +- ADR-0001: Clean Architecture Layering +- ADR-0014: Platform Expansion — Four Pillars +- Issue #531: Platform expansion master tracker +- Issue #537: Cloud/collaboration pillar +- EF Core PostgreSQL provider: https://www.npgsql.org/efcore/ +- Migration runbook: `docs/platform/SQLITE_TO_POSTGRES_MIGRATION_RUNBOOK.md` diff --git a/docs/platform/SQLITE_TO_POSTGRES_MIGRATION_RUNBOOK.md b/docs/platform/SQLITE_TO_POSTGRES_MIGRATION_RUNBOOK.md new file mode 100644 index 00000000..fac0e5bd --- /dev/null +++ b/docs/platform/SQLITE_TO_POSTGRES_MIGRATION_RUNBOOK.md @@ -0,0 +1,383 @@ +# SQLite-to-PostgreSQL Migration Runbook + +**Last updated**: 2026-04-09 +**Related ADR**: ADR-0023 (SQLite-to-PostgreSQL Production Migration Strategy) +**Related issue**: #84 (PLAT-01) + +This runbook provides step-by-step instructions for migrating an existing Taskdeck SQLite database to PostgreSQL. It is intended for operators deploying Taskdeck to a hosted environment. + +> **Current repository state**: this is a preparatory runbook, not a fully executable cutover guide yet. +> The application runtime still hard-wires SQLite in `Taskdeck.Infrastructure.DependencyInjection`, and +> the `AddKnowledgeDocumentsAndFts` migration contains SQLite-only FTS5 SQL. Treat this document as the +> canonical operator checklist for the follow-up implementation work needed to make PostgreSQL migration real. + +--- + +## Prerequisites + +- PostgreSQL 15+ installed or a managed instance provisioned (AWS RDS, Azure Flexible Server, or GCP Cloud SQL) +- `psql` CLI available +- `dotnet` CLI (8.0+) available +- Access to the source SQLite database file (default: `taskdeck.db`) +- Taskdeck application stopped (no active writers to the SQLite database) +- Sufficient disk space for the SQLite database, the exported data, and the target PostgreSQL database + +## Pre-Migration Checklist + +1. **Stop the Taskdeck application** to prevent writes during migration. +2. **Back up the SQLite database file**: + ```bash + cp taskdeck.db taskdeck.db.pre-migration-backup + ``` +3. **Record row counts** for verification (save output for comparison): + ```bash + sqlite3 taskdeck.db <<'SQL' + SELECT 'Users' AS tbl, COUNT(*) FROM Users + UNION ALL SELECT 'Boards', COUNT(*) FROM Boards + UNION ALL SELECT 'Columns', COUNT(*) FROM Columns + UNION ALL SELECT 'Cards', COUNT(*) FROM Cards + UNION ALL SELECT 'Labels', COUNT(*) FROM Labels + UNION ALL SELECT 'CardLabels', COUNT(*) FROM CardLabels + UNION ALL SELECT 'CardComments', COUNT(*) FROM CardComments + UNION ALL SELECT 'BoardAccesses', COUNT(*) FROM BoardAccesses + UNION ALL SELECT 'AuditLogs', COUNT(*) FROM AuditLogs + UNION ALL SELECT 'AutomationProposals', COUNT(*) FROM AutomationProposals + UNION ALL SELECT 'AutomationProposalOperations', COUNT(*) FROM AutomationProposalOperations + UNION ALL SELECT 'ArchiveItems', COUNT(*) FROM ArchiveItems + UNION ALL SELECT 'ChatSessions', COUNT(*) FROM ChatSessions + UNION ALL SELECT 'ChatMessages', COUNT(*) FROM ChatMessages + UNION ALL SELECT 'CommandRuns', COUNT(*) FROM CommandRuns + UNION ALL SELECT 'Notifications', COUNT(*) FROM Notifications + UNION ALL SELECT 'UserPreferences', COUNT(*) FROM UserPreferences + UNION ALL SELECT 'LlmRequests', COUNT(*) FROM LlmRequests + UNION ALL SELECT 'LlmUsageRecords', COUNT(*) FROM LlmUsageRecords + UNION ALL SELECT 'OutboundWebhookSubscriptions', COUNT(*) FROM OutboundWebhookSubscriptions + UNION ALL SELECT 'OutboundWebhookDeliveries', COUNT(*) FROM OutboundWebhookDeliveries + UNION ALL SELECT 'AgentProfiles', COUNT(*) FROM AgentProfiles + UNION ALL SELECT 'AgentRuns', COUNT(*) FROM AgentRuns + UNION ALL SELECT 'KnowledgeDocuments', COUNT(*) FROM KnowledgeDocuments + UNION ALL SELECT 'KnowledgeChunks', COUNT(*) FROM KnowledgeChunks + UNION ALL SELECT 'ExternalLogins', COUNT(*) FROM ExternalLogins + UNION ALL SELECT 'ApiKeys', COUNT(*) FROM ApiKeys + UNION ALL SELECT 'CardCommentMentions', COUNT(*) FROM CardCommentMentions + UNION ALL SELECT 'CommandRunLogs', COUNT(*) FROM CommandRunLogs + UNION ALL SELECT 'AgentRunEvents', COUNT(*) FROM AgentRunEvents + UNION ALL SELECT 'NotificationPreferences', COUNT(*) FROM NotificationPreferences; + SQL + ``` + + **Note**: The `__EFMigrationsHistory` table tracks applied EF Core migrations. It is populated automatically by `dotnet ef database update` in Step 1 and should **not** be exported or imported as data — the schema step handles it. + +4. **Provision the PostgreSQL database**: + ```bash + psql -h -U -c "CREATE DATABASE taskdeck ENCODING 'UTF8' LC_COLLATE 'en_US.UTF-8';" + psql -h -U -c "CREATE USER taskdeck_app WITH PASSWORD '';" + psql -h -U -c "GRANT CONNECT ON DATABASE taskdeck TO taskdeck_app;" + psql -h -U -d taskdeck -c "GRANT USAGE, CREATE ON SCHEMA public TO taskdeck_app;" + psql -h -U -d taskdeck -c "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO taskdeck_app;" + psql -h -U -d taskdeck -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO taskdeck_app;" + ``` + + Use `CREATE` on the schema only while bootstrapping the PostgreSQL schema with EF Core. After the schema is in place, tighten the application user back to DML-only permissions if migrations are handled by an admin/operator identity. + +## Step 1: Apply PostgreSQL Schema via EF Core Migrations + +Prepare the application to target PostgreSQL and apply migrations. + +> **Blocked today**: `Taskdeck.Infrastructure.DependencyInjection.AddInfrastructure()` still calls +> `UseSqlite()` unconditionally. There is no shipped `Taskdeck__DatabaseProvider` switch or runtime +> `UseNpgsql()` path in the application projects yet, so the commands below are follow-up implementation +> steps rather than something the current branch can execute successfully as-is. + +> **Warning — FTS5 migration blocker**: The migration `AddKnowledgeDocumentsAndFts` contains +> raw SQLite-specific SQL (`CREATE VIRTUAL TABLE ... USING fts5`, `CREATE TRIGGER`). These +> statements will fail on PostgreSQL. Before running `dotnet ef database update`, you must +> add provider-conditional guards to that migration: +> +> ```csharp +> if (migrationBuilder.ActiveProvider == "Microsoft.EntityFrameworkCore.Sqlite") +> { +> migrationBuilder.Sql(@"CREATE VIRTUAL TABLE IF NOT EXISTS ..."); +> migrationBuilder.Sql(@"CREATE TRIGGER IF NOT EXISTS ..."); +> } +> ``` +> +> Apply the same guard to the `Down()` method. See the "Full-Text Search Migration Note" +> section below for PostgreSQL FTS setup guidance. + +```bash +# Follow-up implementation required before this step can work: +# 1. Add Npgsql to the runtime application infrastructure path +# 2. Add configuration-based provider selection in AddInfrastructure() +# 3. Guard SQLite-only migration SQL with ActiveProvider checks +# 4. Then run the PostgreSQL schema apply command against the empty database +``` + +Verify the schema was created: +```bash +psql -h -U taskdeck_app -d taskdeck -c "\dt" +``` + +All expected tables should appear. + +## Step 2: Export Data from SQLite + +Export each table to CSV using `sqlite3`: + +```bash +mkdir -p migration-export + +sqlite3 -header -csv taskdeck.db "SELECT * FROM Users;" > migration-export/Users.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM Boards;" > migration-export/Boards.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM Columns;" > migration-export/Columns.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM Cards;" > migration-export/Cards.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM Labels;" > migration-export/Labels.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM CardLabels;" > migration-export/CardLabels.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM CardComments;" > migration-export/CardComments.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM CardCommentMentions;" > migration-export/CardCommentMentions.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM BoardAccesses;" > migration-export/BoardAccesses.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM AuditLogs;" > migration-export/AuditLogs.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM LlmRequests;" > migration-export/LlmRequests.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM AutomationProposals;" > migration-export/AutomationProposals.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM AutomationProposalOperations;" > migration-export/AutomationProposalOperations.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM ArchiveItems;" > migration-export/ArchiveItems.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM ChatSessions;" > migration-export/ChatSessions.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM ChatMessages;" > migration-export/ChatMessages.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM CommandRuns;" > migration-export/CommandRuns.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM CommandRunLogs;" > migration-export/CommandRunLogs.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM Notifications;" > migration-export/Notifications.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM NotificationPreferences;" > migration-export/NotificationPreferences.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM UserPreferences;" > migration-export/UserPreferences.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM OutboundWebhookSubscriptions;" > migration-export/OutboundWebhookSubscriptions.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM OutboundWebhookDeliveries;" > migration-export/OutboundWebhookDeliveries.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM LlmUsageRecords;" > migration-export/LlmUsageRecords.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM AgentProfiles;" > migration-export/AgentProfiles.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM AgentRuns;" > migration-export/AgentRuns.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM AgentRunEvents;" > migration-export/AgentRunEvents.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM KnowledgeDocuments;" > migration-export/KnowledgeDocuments.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM KnowledgeChunks;" > migration-export/KnowledgeChunks.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM ExternalLogins;" > migration-export/ExternalLogins.csv +sqlite3 -header -csv taskdeck.db "SELECT * FROM ApiKeys;" > migration-export/ApiKeys.csv +``` + +## Step 3: Import Data into PostgreSQL + +**Important**: Import tables in dependency order (parents before children) to respect foreign key constraints. + +**Schema note**: EF Core Npgsql creates tables in the `public` schema by default with PascalCase names. Before importing, verify the table names match by running `\dt` in `psql`. If a custom schema was configured, adjust the table names in the import script accordingly. + +```bash +# Order matters: parent tables first, then child tables +TABLES=( + Users + Boards + Columns + Cards + Labels + CardLabels + CardComments + CardCommentMentions + BoardAccesses + AuditLogs + LlmRequests + AutomationProposals + AutomationProposalOperations + ArchiveItems + ChatSessions + ChatMessages + CommandRuns + CommandRunLogs + Notifications + NotificationPreferences + UserPreferences + OutboundWebhookSubscriptions + OutboundWebhookDeliveries + LlmUsageRecords + AgentProfiles + AgentRuns + AgentRunEvents + KnowledgeDocuments + KnowledgeChunks + ExternalLogins + ApiKeys +) + +PGCONN="host= dbname=taskdeck user=taskdeck_app password=" + +for table in "${TABLES[@]}"; do + if [ -f "migration-export/${table}.csv" ] && [ -s "migration-export/${table}.csv" ]; then + echo "Importing ${table}..." + # Use \COPY to import CSV (runs client-side, no server file access needed) + psql "$PGCONN" -c "\\COPY \"${table}\" FROM 'migration-export/${table}.csv' WITH (FORMAT csv, HEADER true)" + if [ $? -ne 0 ]; then + echo "ERROR: Failed to import ${table}. Stopping." >&2 + exit 1 + fi + else + echo "Skipping ${table} (empty or missing CSV)." + fi +done + +``` + +Managed PostgreSQL services often do not allow blanket trigger disabling, and `session_replication_role` +is generally too privileged for an application migration user. Keep constraints enabled, import in +dependency order, and treat any FK failure as a migration defect to fix before retrying. + +**GUID column handling**: SQLite stores GUIDs as text strings. PostgreSQL with Npgsql maps `Guid` properties to the native `uuid` type. EF Core's Npgsql provider accepts standard UUID text format (`xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`), so CSV import should work directly. If you encounter format errors, verify the GUID format in the CSV matches PostgreSQL's expected input. + +**DateTimeOffset handling**: SQLite stores `DateTimeOffset` as ISO 8601 text strings. PostgreSQL stores them as `timestamptz`. The CSV import should parse ISO 8601 strings correctly. If timezone information is missing from SQLite values, PostgreSQL will assume UTC. + +## Step 4: Verify Data Integrity + +Run row-count verification against PostgreSQL and compare with the pre-migration counts from the checklist: + +```bash +psql "$PGCONN" <<'SQL' +SELECT 'Users' AS tbl, COUNT(*) FROM "Users" +UNION ALL SELECT 'Boards', COUNT(*) FROM "Boards" +UNION ALL SELECT 'Columns', COUNT(*) FROM "Columns" +UNION ALL SELECT 'Cards', COUNT(*) FROM "Cards" +UNION ALL SELECT 'Labels', COUNT(*) FROM "Labels" +UNION ALL SELECT 'CardLabels', COUNT(*) FROM "CardLabels" +UNION ALL SELECT 'CardComments', COUNT(*) FROM "CardComments" +UNION ALL SELECT 'BoardAccesses', COUNT(*) FROM "BoardAccesses" +UNION ALL SELECT 'AuditLogs', COUNT(*) FROM "AuditLogs" +UNION ALL SELECT 'AutomationProposals', COUNT(*) FROM "AutomationProposals" +UNION ALL SELECT 'AutomationProposalOperations', COUNT(*) FROM "AutomationProposalOperations" +UNION ALL SELECT 'ArchiveItems', COUNT(*) FROM "ArchiveItems" +UNION ALL SELECT 'ChatSessions', COUNT(*) FROM "ChatSessions" +UNION ALL SELECT 'ChatMessages', COUNT(*) FROM "ChatMessages" +UNION ALL SELECT 'CommandRuns', COUNT(*) FROM "CommandRuns" +UNION ALL SELECT 'Notifications', COUNT(*) FROM "Notifications" +UNION ALL SELECT 'UserPreferences', COUNT(*) FROM "UserPreferences" +UNION ALL SELECT 'LlmRequests', COUNT(*) FROM "LlmRequests" +UNION ALL SELECT 'LlmUsageRecords', COUNT(*) FROM "LlmUsageRecords" +UNION ALL SELECT 'OutboundWebhookSubscriptions', COUNT(*) FROM "OutboundWebhookSubscriptions" +UNION ALL SELECT 'OutboundWebhookDeliveries', COUNT(*) FROM "OutboundWebhookDeliveries" +UNION ALL SELECT 'AgentProfiles', COUNT(*) FROM "AgentProfiles" +UNION ALL SELECT 'AgentRuns', COUNT(*) FROM "AgentRuns" +UNION ALL SELECT 'KnowledgeDocuments', COUNT(*) FROM "KnowledgeDocuments" +UNION ALL SELECT 'KnowledgeChunks', COUNT(*) FROM "KnowledgeChunks" +UNION ALL SELECT 'ExternalLogins', COUNT(*) FROM "ExternalLogins" +UNION ALL SELECT 'ApiKeys', COUNT(*) FROM "ApiKeys" +UNION ALL SELECT 'NotificationPreferences', COUNT(*) FROM "NotificationPreferences" +UNION ALL SELECT 'CommandRunLogs', COUNT(*) FROM "CommandRunLogs" +UNION ALL SELECT 'CardCommentMentions', COUNT(*) FROM "CardCommentMentions" +UNION ALL SELECT 'AgentRunEvents', COUNT(*) FROM "AgentRunEvents"; +SQL +``` + +Verify foreign key integrity: +```bash +psql "$PGCONN" <<'SQL' +-- Cards reference valid columns +SELECT COUNT(*) AS orphaned_cards +FROM "Cards" c +LEFT JOIN "Columns" col ON c."ColumnId" = col."Id" +WHERE col."Id" IS NULL; + +-- Columns reference valid boards +SELECT COUNT(*) AS orphaned_columns +FROM "Columns" col +LEFT JOIN "Boards" b ON col."BoardId" = b."Id" +WHERE b."Id" IS NULL; + +-- BoardAccesses reference valid boards and users +SELECT COUNT(*) AS orphaned_accesses +FROM "BoardAccesses" ba +LEFT JOIN "Boards" b ON ba."BoardId" = b."Id" +LEFT JOIN "Users" u ON ba."UserId" = u."Id" +WHERE b."Id" IS NULL OR u."Id" IS NULL; + +-- AutomationProposalOperations reference valid proposals +SELECT COUNT(*) AS orphaned_operations +FROM "AutomationProposalOperations" apo +LEFT JOIN "AutomationProposals" ap ON apo."ProposalId" = ap."Id" +WHERE ap."Id" IS NULL; +SQL +``` + +All orphan counts must be **zero**. If any are non-zero, the migration has data integrity issues — do not proceed to Step 5. + +## Step 5: Smoke Test the Application + +1. Configure the application for PostgreSQL: + ```bash + # Requires the follow-up runtime provider switch described in Step 1. + # The current codebase cannot run the API against PostgreSQL yet. + ``` +2. Start the application: + ```bash + dotnet run --project backend/src/Taskdeck.Api + ``` +3. Verify core operations: + - [ ] Login with an existing user + - [ ] List boards (GET /api/boards) + - [ ] Open a board with cards and columns + - [ ] Create a new card + - [ ] Move a card between columns + - [ ] Submit a capture and verify it appears in the inbox + - [ ] Check audit log entries are being written + - [ ] Verify chat session loads with history + +## Rollback Procedure + +If the migration fails or the application does not function correctly against PostgreSQL: + +1. **Stop the application** targeting PostgreSQL. +2. **Restore the SQLite configuration**: + ```bash + export ConnectionStrings__DefaultConnection="Data Source=taskdeck.db" + ``` +3. **Restore the SQLite backup** if the original file was modified: + ```bash + cp taskdeck.db.pre-migration-backup taskdeck.db + ``` +4. **Restart the application** — it will use the SQLite database. +5. **Investigate the failure** using logs and the integrity verification queries above. +6. The PostgreSQL database can be dropped and recreated for a fresh retry: + ```bash + psql -h -U -c "DROP DATABASE taskdeck;" + ``` + +## Known Provider Differences + +These differences are handled by EF Core's provider abstraction but are worth noting: + +| Concern | SQLite | PostgreSQL | +|---------|--------|------------| +| GUID storage | Text (string) | Native `uuid` type | +| DateTimeOffset | ISO 8601 text | `timestamptz` | +| String comparison | Case-sensitive by default | Case-sensitive by default (use `ILIKE` for insensitive) | +| Auto-increment | `AUTOINCREMENT` keyword | `SERIAL` / `GENERATED ALWAYS AS IDENTITY` | +| JSON columns | Text with no validation | `jsonb` with indexing support | +| Full-text search | FTS5 virtual tables | `tsvector` / `tsquery` (requires different setup) | +| Concurrency | Database-level write lock | Row-level locking | +| Max connections | Single writer | Configurable (default 100) | + +**Sequence note**: the current Taskdeck schema uses GUID primary keys, so no PostgreSQL sequence reset +step is needed after import. If future tables introduce integer identity columns, add `setval(...)` +calls after the CSV import to advance those sequences to the current max values. + +## Full-Text Search Migration Note + +The current `KnowledgeDocuments` and `KnowledgeChunks` tables use SQLite FTS5 for full-text search. PostgreSQL uses a different FTS mechanism (`tsvector`/`tsquery`). The `IKnowledgeSearchService` interface abstracts this, so the migration requires a PostgreSQL-specific implementation of that interface — no domain or application layer changes. + +Key details for the FTS migration: + +- **`KnowledgeDocumentsFts`** is a SQLite FTS5 virtual table. It does **not** exist in the EF Core model and should **not** be exported or imported. It is populated by a SQLite trigger (`KnowledgeDocuments_ai`) that also does not exist in PostgreSQL. +- **Do not export** `KnowledgeDocumentsFts` — it is not a regular table and `SELECT *` on an FTS5 table may produce unexpected results. +- For PostgreSQL, full-text search can be implemented using `tsvector`/`tsquery` columns with GIN indexes, or using `pg_trgm` for simpler similarity-based search. The PostgreSQL-specific `IKnowledgeSearchService` implementation is a separate work item. +- Until the PostgreSQL FTS implementation is built, knowledge document search will be non-functional on PostgreSQL. The `KnowledgeDocuments` and `KnowledgeChunks` data tables themselves will migrate normally. + +## Security Considerations + +- **Never store database credentials in source control.** Use environment variables, secrets managers (AWS Secrets Manager, Azure Key Vault), or mounted secret files. +- Prefer `~/.pgpass`, `PGPASSFILE`, or a secrets-mounted appsettings file over inline passwords in shell history when running `psql` commands. +- **Use TLS for PostgreSQL connections** in production (`SslMode=Require` in the connection string). +- **Restrict the `taskdeck_app` database user** to only the permissions needed (SELECT, INSERT, UPDATE, DELETE on application tables). Do not grant `SUPERUSER` or `CREATEDB`. +- **The migration-export directory contains all application data** including password hashes. Delete it securely after a successful migration: + ```bash + rm -rf migration-export/ + ```