From cfa1f948383d0aa6d2d8528ce620eb87c402c290 Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Fri, 13 Feb 2026 13:22:26 +0100 Subject: [PATCH 01/19] Add backup manifest types and folder scanner - Add BackupManifest, ManifestEntry, ScannedFile types - Add scanBackupFolder() to recursively read markdown + assets - Add readManifest() and writeManifest() helpers - Add transformContentForImport() for reverse asset reference conversion - Add tests for scanner, manifest, and content transform --- src/lib/backup-sync.test.ts | 386 ++++++++++++++++++++++++++++++++++++ src/lib/backup-sync.ts | 134 +++++++++++++ 2 files changed, 520 insertions(+) create mode 100644 src/lib/backup-sync.test.ts diff --git a/src/lib/backup-sync.test.ts b/src/lib/backup-sync.test.ts new file mode 100644 index 0000000..a3c8254 --- /dev/null +++ b/src/lib/backup-sync.test.ts @@ -0,0 +1,386 @@ +import { describe, it, expect } from "vitest" +import { + scanBackupFolder, + transformContentForImport, + readManifest, + writeManifest, + type BackupManifest, +} from "./backup-sync" + +// ============================================================================= +// Test Helpers +// ============================================================================= + +function createMockFile(content: string, lastModified = Date.now()): File { + return new File([content], "test.md", { lastModified }) +} + +function createMockBlob(content: string, type = "image/png"): Blob { + return new Blob([content], { type }) +} + +// Mock FileSystemDirectoryHandle for testing +class MockDirectoryHandle implements FileSystemDirectoryHandle { + kind = "directory" as const + name: string + private children = new Map< + string, + FileSystemFileHandle | FileSystemDirectoryHandle + >() + + constructor(name: string) { + this.name = name + } + + private fileContents = new Map() + + addFile(name: string, file: File) { + let self = this + let mockHandle: FileSystemFileHandle = { + kind: "file", + name, + async getFile() { + // Check if there's saved content from writeManifest + let content = self.fileContents.get(name) + if (content !== undefined) { + return new File([content], name) + } + return file + }, + async createWritable() { + return { + async write(data: string | Blob) { + let content = typeof data === "string" ? data : await data.text() + self.fileContents.set(name, content) + }, + async close() {}, + } + }, + } as unknown as FileSystemFileHandle + this.children.set(name, mockHandle) + } + + addDirectory(name: string, dir: FileSystemDirectoryHandle) { + this.children.set(name, dir) + } + + entries(): AsyncIterableIterator<[string, FileSystemHandle]> { + let iter = this.children.entries() + return { + async next() { + let result = iter.next() + return result.done + ? { done: true, value: undefined } + : { done: false, value: result.value } + }, + [Symbol.asyncIterator]() { + return this + }, + } + } + + async getFileHandle( + name: string, + options?: { create?: boolean }, + ): Promise { + let self = this + let handle = this.children.get(name) + if (!handle || handle.kind !== "file") { + if (options?.create) { + let mockHandle: FileSystemFileHandle = { + kind: "file", + name, + async getFile() { + let content = self.fileContents.get(name) + return new File([content ?? ""], name) + }, + async createWritable() { + return { + async write(data: string | Blob) { + let content = + typeof data === "string" ? data : await data.text() + self.fileContents.set(name, content) + }, + async close() {}, + } + }, + } as unknown as FileSystemFileHandle + this.children.set(name, mockHandle) + return mockHandle + } + throw new Error(`File not found: ${name}`) + } + return handle as FileSystemFileHandle + } + + async getDirectoryHandle( + name: string, + options?: { create?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!handle || handle.kind !== "directory") { + if (options?.create) { + let newDir = new MockDirectoryHandle(name) + this.children.set(name, newDir) + return newDir + } + throw new Error(`Directory not found: ${name}`) + } + return handle as FileSystemDirectoryHandle + } + + removeEntry(): Promise { + return Promise.resolve() + } + + resolve(): Promise { + return Promise.resolve([this.name]) + } + + isSameEntry(other: FileSystemHandle): Promise { + return Promise.resolve(this.name === other.name) + } + + queryPermission(): Promise<"granted"> { + return Promise.resolve("granted") + } + + requestPermission(): Promise<"granted"> { + return Promise.resolve("granted") + } + + get [Symbol.toStringTag](): string { + return "FileSystemDirectoryHandle" + } +} + +// ============================================================================= +// Transform Content for Import +// ============================================================================= + +describe("transformContentForImport", () => { + it("transforms asset paths to asset: references", () => { + let assetFiles = new Map([ + ["asset1", "image.png"], + ["asset2", "photo.jpg"], + ]) + + let content = `# Test + +![Screenshot](assets/image.png) +![Photo](assets/photo.jpg) +` + + let result = transformContentForImport(content, assetFiles) + + expect(result).toContain("![Screenshot](asset:asset1)") + expect(result).toContain("![Photo](asset:asset2)") + }) + + it("preserves non-asset references", () => { + let assetFiles = new Map([["asset1", "image.png"]]) + + let content = `# Test + +![External](https://example.com/img.png) +![Local](./local/image.png) +![Asset](assets/image.png) +` + + let result = transformContentForImport(content, assetFiles) + + expect(result).toContain("https://example.com/img.png") + expect(result).toContain("./local/image.png") + expect(result).toContain("asset:asset1") + }) + + it("keeps unknown asset paths unchanged", () => { + let assetFiles = new Map([["asset1", "image.png"]]) + + let content = `![Unknown](assets/unknown.jpg)` + + let result = transformContentForImport(content, assetFiles) + + expect(result).toBe(`![Unknown](assets/unknown.jpg)`) + }) +}) + +// ============================================================================= +// Manifest Read/Write +// ============================================================================= + +describe("readManifest", () => { + it("returns null when manifest file does not exist", async () => { + let root = new MockDirectoryHandle("root") + + let result = await readManifest(root) + + expect(result).toBeNull() + }) + + it("reads and parses valid manifest", async () => { + let root = new MockDirectoryHandle("root") + let manifest: BackupManifest = { + version: 1, + entries: [ + { + docId: "doc1", + relativePath: "Test.md", + contentHash: "abc123", + lastSyncedAt: new Date().toISOString(), + assets: [], + }, + ], + lastSyncAt: new Date().toISOString(), + } + + root.addFile( + ".alkalye-manifest.json", + new File([JSON.stringify(manifest)], ".alkalye-manifest.json"), + ) + + let result = await readManifest(root) + + expect(result).not.toBeNull() + expect(result!.version).toBe(1) + expect(result!.entries).toHaveLength(1) + expect(result!.entries[0].docId).toBe("doc1") + }) + + it("returns null for invalid version", async () => { + let root = new MockDirectoryHandle("root") + let invalidManifest = { version: 2, entries: [] } + + root.addFile( + ".alkalye-manifest.json", + new File([JSON.stringify(invalidManifest)], ".alkalye-manifest.json"), + ) + + let result = await readManifest(root) + + expect(result).toBeNull() + }) +}) + +describe("writeManifest", () => { + it("writes manifest to directory", async () => { + let root = new MockDirectoryHandle("root") + let manifest: BackupManifest = { + version: 1, + entries: [ + { + docId: "doc1", + relativePath: "Test.md", + contentHash: "abc123", + lastSyncedAt: "2024-01-01T00:00:00Z", + assets: [], + }, + ], + lastSyncAt: "2024-01-01T00:00:00Z", + } + + await writeManifest(root, manifest) + + let written = await readManifest(root) + expect(written).not.toBeNull() + expect(written!.entries[0].docId).toBe("doc1") + }) +}) + +// ============================================================================= +// Scan Backup Folder +// ============================================================================= + +describe("scanBackupFolder", () => { + it("scans markdown files at root level", async () => { + let root = new MockDirectoryHandle("root") + root.addFile("Test.md", createMockFile("# Test Content")) + root.addFile("Another.md", createMockFile("# Another")) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(2) + let names = files.map(f => f.name).sort() + expect(names).toEqual(["Another", "Test"]) + }) + + it("recursively scans nested directories", async () => { + let root = new MockDirectoryHandle("root") + let workDir = new MockDirectoryHandle("work") + let notesDir = new MockDirectoryHandle("notes") + + workDir.addDirectory("notes", notesDir) + root.addDirectory("work", workDir) + + root.addFile("Root.md", createMockFile("# Root")) + workDir.addFile("Work.md", createMockFile("# Work")) + notesDir.addFile("Notes.md", createMockFile("# Notes")) + + let files = await scanBackupFolder(root) + + let paths = files.map(f => f.relativePath).sort() + expect(paths).toEqual(["Root.md", "work/Work.md", "work/notes/Notes.md"]) + }) + + it("collects assets from assets folders", async () => { + let root = new MockDirectoryHandle("root") + let docDir = new MockDirectoryHandle("My Doc") + + root.addDirectory("My Doc", docDir) + let assetsDir = new MockDirectoryHandle("assets") + docDir.addDirectory("assets", assetsDir) + + docDir.addFile( + "My Doc.md", + createMockFile("# My Doc\n\n![Image](assets/photo.png)"), + ) + assetsDir.addFile( + "photo.png", + new File([createMockBlob("image data")], "photo.png"), + ) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(1) + expect(files[0].assets).toHaveLength(1) + expect(files[0].assets[0].name).toBe("photo.png") + }) + + it("skips dot directories", async () => { + let root = new MockDirectoryHandle("root") + let hiddenDir = new MockDirectoryHandle(".hidden") + + root.addDirectory(".hidden", hiddenDir) + hiddenDir.addFile("Hidden.md", createMockFile("# Hidden")) + root.addFile("Visible.md", createMockFile("# Visible")) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(1) + expect(files[0].name).toBe("Visible") + }) + + it("skips manifest file", async () => { + let root = new MockDirectoryHandle("root") + root.addFile( + ".alkalye-manifest.json", + new File(['{"version":1}'], ".alkalye-manifest.json"), + ) + root.addFile("Test.md", createMockFile("# Test")) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(1) + expect(files[0].name).toBe("Test") + }) + + it("captures lastModified timestamp", async () => { + let root = new MockDirectoryHandle("root") + let timestamp = 1234567890000 + root.addFile("Test.md", createMockFile("# Test", timestamp)) + + let files = await scanBackupFolder(root) + + expect(files[0].lastModified).toBe(timestamp) + }) +}) diff --git a/src/lib/backup-sync.ts b/src/lib/backup-sync.ts index ca0ad99..b104bca 100644 --- a/src/lib/backup-sync.ts +++ b/src/lib/backup-sync.ts @@ -4,9 +4,14 @@ export { computeDocLocations, transformContentForBackup, computeExpectedStructure, + transformContentForImport, + scanBackupFolder, type BackupDoc, type DocLocation, type ExpectedStructure, + type BackupManifest, + type ManifestEntry, + type ScannedFile, } interface BackupDoc { @@ -29,6 +34,28 @@ interface ExpectedStructure { expectedFiles: Map> } +interface ManifestEntry { + docId: string + relativePath: string + contentHash: string + lastSyncedAt: string + assets: { name: string; hash: string }[] +} + +interface BackupManifest { + version: 1 + entries: ManifestEntry[] + lastSyncAt: string +} + +interface ScannedFile { + relativePath: string + name: string + content: string + assets: { name: string; blob: Blob }[] + lastModified: number +} + function computeDocLocations(docs: BackupDoc[]): Map { let docLocations = new Map() let usedNames = new Map>() // parentPath -> used names (lowercase) @@ -102,6 +129,26 @@ function transformContentForBackup( ) } +function transformContentForImport( + content: string, + assetFiles: Map, +): string { + // Transform local asset paths back to asset: references + return content.replace( + /!\[([^\]]*)\]\(assets\/([^)]+)\)/g, + (match, alt, assetFilename) => { + // Find asset ID by filename + for (let [id, filename] of assetFiles) { + if (filename === assetFilename) { + return `![${alt}](asset:${id})` + } + } + // If not found, keep the original local path (might be a manual addition) + return match + }, + ) +} + function computeExpectedStructure( docs: BackupDoc[], docLocations: Map, @@ -135,3 +182,90 @@ function computeExpectedStructure( return { expectedPaths, expectedFiles } } + +async function scanBackupFolder( + handle: FileSystemDirectoryHandle, +): Promise { + let files: ScannedFile[] = [] + + async function scanDir( + dir: FileSystemDirectoryHandle, + relativePath: string, + ): Promise { + for await (let [name, handle] of dir.entries()) { + let entryPath = relativePath ? `${relativePath}/${name}` : name + + if (handle.kind === "directory") { + // Skip dot directories and special directories + if (name.startsWith(".")) continue + let subDir = await dir.getDirectoryHandle(name) + await scanDir(subDir, entryPath) + } else if (handle.kind === "file" && name.endsWith(".md")) { + // Skip manifest file + if (name === ".alkalye-manifest.json") continue + + let fileHandle = await dir.getFileHandle(name) + let file = await fileHandle.getFile() + let content = await file.text() + let lastModified = file.lastModified + + // Check for assets folder + let assets: { name: string; blob: Blob }[] = [] + try { + let assetsDir = await dir.getDirectoryHandle("assets") + for await (let [assetName, assetHandle] of assetsDir.entries()) { + if (assetHandle.kind === "file" && !assetName.startsWith(".")) { + let assetFileHandle = await assetsDir.getFileHandle(assetName) + let assetFile = await assetFileHandle.getFile() + assets.push({ name: assetName, blob: assetFile }) + } + } + } catch { + // No assets folder + } + + files.push({ + relativePath: entryPath, + name: name.replace(/\.md$/, ""), + content, + assets, + lastModified, + }) + } + } + } + + await scanDir(handle, "") + return files +} + +async function readManifest( + handle: FileSystemDirectoryHandle, +): Promise { + try { + let fileHandle = await handle.getFileHandle(".alkalye-manifest.json") + let file = await fileHandle.getFile() + let text = await file.text() + let parsed = JSON.parse(text) + if (parsed.version === 1) { + return parsed as BackupManifest + } + return null + } catch { + return null + } +} + +export { readManifest, writeManifest } + +async function writeManifest( + handle: FileSystemDirectoryHandle, + manifest: BackupManifest, +): Promise { + let fileHandle = await handle.getFileHandle(".alkalye-manifest.json", { + create: true, + }) + let writable = await fileHandle.createWritable() + await writable.write(JSON.stringify(manifest, null, 2)) + await writable.close() +} From 789d472f7a60759be042335f5e1e39e985589571 Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Fri, 13 Feb 2026 13:23:25 +0100 Subject: [PATCH 02/19] Add bidirectional sync setting to backup store - Add bidirectional flag to BackupState (default true for new installs) - Add setBidirectional setter and reset handling - Add lastPullAt tracking for UI display - Add BACKUP_PULL_INTERVAL_MS constant (20s) --- src/lib/backup.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx index 0aa5dbc..ed7403e 100644 --- a/src/lib/backup.tsx +++ b/src/lib/backup.tsx @@ -50,16 +50,21 @@ declare global { } let BACKUP_DEBOUNCE_MS = 5000 +let BACKUP_PULL_INTERVAL_MS = 20000 let HANDLE_STORAGE_KEY = "backup-directory-handle" interface BackupState { enabled: boolean + bidirectional: boolean directoryName: string | null lastBackupAt: string | null + lastPullAt: string | null lastError: string | null setEnabled: (enabled: boolean) => void + setBidirectional: (bidirectional: boolean) => void setDirectoryName: (name: string | null) => void setLastBackupAt: (date: string | null) => void + setLastPullAt: (date: string | null) => void setLastError: (error: string | null) => void reset: () => void } @@ -68,18 +73,24 @@ let useBackupStore = create()( persist( set => ({ enabled: false, + bidirectional: true, directoryName: null, lastBackupAt: null, + lastPullAt: null, lastError: null, setEnabled: enabled => set({ enabled }), + setBidirectional: bidirectional => set({ bidirectional }), setDirectoryName: directoryName => set({ directoryName }), setLastBackupAt: lastBackupAt => set({ lastBackupAt }), + setLastPullAt: lastPullAt => set({ lastPullAt }), setLastError: lastError => set({ lastError }), reset: () => set({ enabled: false, + bidirectional: true, directoryName: null, lastBackupAt: null, + lastPullAt: null, lastError: null, }), }), From 6233b9ef88af5f53a2b69b6147fab9e8e360c842 Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Fri, 13 Feb 2026 13:31:54 +0100 Subject: [PATCH 03/19] Add bidirectional sync core functions and personal backup pull loop - Add hashContent(), syncFromBackup(), createDocFromFile(), updateDocFromFile() - Add pull loop with visibility detection and 20s interval - Update BackupSubscriber to include bidirectional sync (bidirectional flag) - Add setLastPullAt tracking for UI display - Support creating/updating/deleting docs from filesystem changes --- src/lib/backup.tsx | 285 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 280 insertions(+), 5 deletions(-) diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx index ed7403e..7887395 100644 --- a/src/lib/backup.tsx +++ b/src/lib/backup.tsx @@ -2,9 +2,17 @@ import { useState, useEffect, useRef } from "react" import { create } from "zustand" import { persist } from "zustand/middleware" import { useAccount, useCoState } from "jazz-tools/react" -import { co, type ResolveQuery } from "jazz-tools" +import { co, type ResolveQuery, Group, FileStream } from "jazz-tools" +import { createImage } from "jazz-tools/media" import { FolderOpen, AlertCircle } from "lucide-react" -import { UserAccount, Document, Space } from "@/schema" +import { + UserAccount, + Document, + Space, + Asset, + ImageAsset, + VideoAsset, +} from "@/schema" import { getDocumentTitle } from "@/lib/document-utils" import { getPath } from "@/editor/frontmatter" import { Button } from "@/components/ui/button" @@ -13,7 +21,11 @@ import { computeDocLocations, transformContentForBackup, computeExpectedStructure, + scanBackupFolder, + readManifest, + transformContentForImport, type BackupDoc, + type ScannedFile, } from "@/lib/backup-sync" export { @@ -50,7 +62,7 @@ declare global { } let BACKUP_DEBOUNCE_MS = 5000 -let BACKUP_PULL_INTERVAL_MS = 20000 +let _BACKUP_PULL_INTERVAL_MS = 20000 let HANDLE_STORAGE_KEY = "backup-directory-handle" interface BackupState { @@ -103,6 +115,9 @@ type LoadedDocument = co.loaded< { content: true; assets: { $each: { image: true; video: true } } } > +type DocumentList = co.loaded>> +type Account = co.loaded + let backupQuery = { root: { documents: { @@ -113,12 +128,21 @@ let backupQuery = { } as const satisfies ResolveQuery function BackupSubscriber() { - let { enabled, setLastBackupAt, setLastError, setEnabled, setDirectoryName } = - useBackupStore() + let { + enabled, + bidirectional, + setLastBackupAt, + setLastPullAt, + setLastError, + setEnabled, + setDirectoryName, + } = useBackupStore() let me = useAccount(UserAccount, { resolve: backupQuery }) let debounceRef = useRef | null>(null) let lastContentHashRef = useRef("") + let pullIntervalRef = useRef | null>(null) + // Push to filesystem (backup) useEffect(() => { if (!enabled || !me.$isLoaded) return @@ -165,6 +189,50 @@ function BackupSubscriber() { } }, [enabled, me, setLastBackupAt, setLastError, setEnabled, setDirectoryName]) + // Pull from filesystem (import changes) + useEffect(() => { + if (!enabled || !bidirectional || !me.$isLoaded) return + + let docs = me.root?.documents + if (!docs?.$isLoaded) return + + async function doPull() { + try { + let handle = await getBackupHandle() + if (!handle) return + + if (!docs.$isLoaded) return + let result = await syncFromBackup(handle, docs as DocumentList, true) + if (result.errors.length > 0) { + console.warn("Backup pull errors:", result.errors) + } + + setLastPullAt(new Date().toISOString()) + } catch (e) { + console.error("Backup pull failed:", e) + } + } + + // Pull on mount and visibility change + doPull() + + let handleVisibility = () => { + if (document.visibilityState === "visible") { + doPull() + } + } + + document.addEventListener("visibilitychange", handleVisibility) + + // Set up interval for periodic pull + pullIntervalRef.current = setInterval(doPull, _BACKUP_PULL_INTERVAL_MS) + + return () => { + document.removeEventListener("visibilitychange", handleVisibility) + if (pullIntervalRef.current) clearInterval(pullIntervalRef.current) + } + }, [enabled, bidirectional, me, setLastPullAt]) + return null } @@ -830,3 +898,210 @@ function getSpacesWithBackup( } return spaceIds } + +// ============================================================================= +// Bidirectional Sync - Pull from filesystem +// ============================================================================= + +async function hashContent(content: string): Promise { + // Simple hash using built-in crypto + let encoder = new TextEncoder() + let data = encoder.encode(content) + let hashBuffer = await crypto.subtle.digest("SHA-256", data) + let hashArray = Array.from(new Uint8Array(hashBuffer)) + return hashArray + .map(b => b.toString(16).padStart(2, "0")) + .join("") + .slice(0, 16) +} + +async function syncFromBackup( + handle: FileSystemDirectoryHandle, + targetDocs: DocumentList, + canWrite: boolean, +): Promise<{ + created: number + updated: number + deleted: number + errors: string[] +}> { + let result = { created: 0, updated: 0, deleted: 0, errors: [] as string[] } + + let manifest = await readManifest(handle) + let scannedFiles = await scanBackupFolder(handle) + let listOwner = targetDocs.$jazz.owner + + // Build maps for lookup + let manifestByPath = new Map( + manifest?.entries.map(e => [e.relativePath, e]) ?? [], + ) + let scannedByPath = new Map(scannedFiles.map(f => [f.relativePath, f])) + + // Process new and updated files + for (let file of scannedFiles) { + try { + let contentHash = await hashContent(file.content) + let manifestEntry = manifestByPath.get(file.relativePath) + + if (!manifestEntry) { + // New file - create document + if (!canWrite) { + result.errors.push(`Cannot create ${file.name}: no write permission`) + continue + } + await createDocFromFile(file, targetDocs, listOwner) + result.created++ + } else if (manifestEntry.contentHash !== contentHash) { + // File changed - update document + if (!canWrite) { + result.errors.push(`Cannot update ${file.name}: no write permission`) + continue + } + await updateDocFromFile( + file, + manifestEntry.docId, + targetDocs, + listOwner, + ) + result.updated++ + } + } catch (err) { + result.errors.push( + `Failed to process ${file.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, + ) + } + } + + // Handle deletions (files in manifest but not on disk) + if (manifest && canWrite) { + for (let entry of manifest.entries) { + if (!scannedByPath.has(entry.relativePath)) { + try { + let doc = targetDocs.find(d => d?.$jazz.id === entry.docId) + if (doc?.$isLoaded && !doc.deletedAt) { + // Soft delete + doc.$jazz.set("deletedAt", new Date()) + doc.$jazz.set("updatedAt", new Date()) + result.deleted++ + } + } catch (err) { + result.errors.push( + `Failed to delete ${entry.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, + ) + } + } + } + } + + return result +} + +async function createDocFromFile( + file: ScannedFile, + targetDocs: DocumentList, + listOwner: Group | Account, +): Promise { + // Transform asset references back to asset: format + let assetFiles = new Map() + for (let asset of file.assets) { + let id = crypto.randomUUID() + assetFiles.set(id, asset.name) + } + let content = transformContentForImport(file.content, assetFiles) + + // Create doc-specific group with list owner as parent + let docGroup = Group.create() + if (listOwner instanceof Group) { + docGroup.addMember(listOwner) + } + + let now = new Date() + + // Create assets + let docAssets: co.loaded[] = [] + for (let assetFile of file.assets) { + let isVideo = assetFile.blob.type.startsWith("video/") + let id = [...assetFiles.entries()].find( + ([, name]) => name === assetFile.name, + )?.[0] + if (!id) continue + + if (isVideo) { + let video = await FileStream.createFromBlob(assetFile.blob, { + owner: docGroup, + }) + let asset = VideoAsset.create( + { + type: "video", + name: assetFile.name.replace(/\.[^.]+$/, ""), + video, + mimeType: "video/mp4", + createdAt: now, + }, + docGroup, + ) + docAssets.push(asset) + } else { + let image = await createImage(assetFile.blob, { + owner: docGroup, + maxSize: 2048, + }) + let asset = ImageAsset.create( + { + type: "image", + name: assetFile.name.replace(/\.[^.]+$/, ""), + image, + createdAt: now, + }, + docGroup, + ) + docAssets.push(asset) + } + } + + let newDoc = Document.create( + { + version: 1, + content: co.plainText().create(content, docGroup), + assets: + docAssets.length > 0 + ? co.list(Asset).create(docAssets, docGroup) + : undefined, + createdAt: now, + updatedAt: now, + }, + docGroup, + ) + + targetDocs.$jazz.push(newDoc) +} + +async function updateDocFromFile( + file: ScannedFile, + docId: string, + targetDocs: DocumentList, + listOwner: Group | Account, +): Promise { + let doc = targetDocs.find( + (d): d is LoadedDocument => d?.$isLoaded === true && d.$jazz.id === docId, + ) + if (!doc || !doc.content?.$isLoaded) { + // Doc doesn't exist or content not loaded, treat as create + await createDocFromFile(file, targetDocs, listOwner) + return + } + + // Update content + let assetFiles = new Map() + for (let asset of file.assets) { + let id = crypto.randomUUID() + assetFiles.set(id, asset.name) + } + let content = transformContentForImport(file.content, assetFiles) + + doc.content.$jazz.applyDiff(content) + doc.$jazz.set("updatedAt", new Date()) + + // TODO: Handle asset updates (add/remove/replace assets) + // For now, we just update the content. Asset changes require more complex logic. +} From daa840267fe5f1ca350c39c029f04a22340254a8 Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Fri, 13 Feb 2026 13:37:33 +0100 Subject: [PATCH 04/19] Add bidirectional sync to SpaceBackupSubscriber - Add pull loop to SpaceBackupSubscriber with visibility detection - Space backups always bidirectional (respects user permissions for writes) - Check admin/writer role before allowing sync-from-backup writes - Add periodic pull with 20s interval - Remove unused lastPullAt state variable --- src/lib/backup.tsx | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx index 7887395..86c0a61 100644 --- a/src/lib/backup.tsx +++ b/src/lib/backup.tsx @@ -572,6 +572,7 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) let debounceRef = useRef | null>(null) let lastContentHashRef = useRef("") + let pullIntervalRef = useRef | null>(null) // Load space with documents let space = useCoState(Space, spaceId as Parameters[1], { @@ -583,6 +584,7 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { }, }) + // Push to filesystem (backup) useEffect(() => { // Skip if no backup folder configured if (!directoryName) return @@ -627,6 +629,60 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { } }, [directoryName, space, spaceId, setDirectoryName]) + // Pull from filesystem (import changes) - always enabled for space backups + useEffect(() => { + // Skip if no backup folder configured + if (!directoryName) return + // Skip if space not loaded + if (!space?.$isLoaded || !space.documents?.$isLoaded) return + + let docs = space.documents + + // Check permissions for writing + let spaceGroup = + space.$jazz.owner instanceof Group ? space.$jazz.owner : null + let canWrite = + spaceGroup?.myRole() === "admin" || spaceGroup?.myRole() === "writer" + + async function doPull() { + try { + let handle = await getSpaceBackupHandle(spaceId) + if (!handle) return + + if (!docs.$isLoaded) return + let result = await syncFromBackup( + handle, + docs as DocumentList, + canWrite, + ) + if (result.errors.length > 0) { + console.warn(`Space ${spaceId} pull errors:`, result.errors) + } + } catch (e) { + console.error(`Space backup pull failed for ${spaceId}:`, e) + } + } + + // Pull on mount and visibility change + doPull() + + let handleVisibility = () => { + if (document.visibilityState === "visible") { + doPull() + } + } + + document.addEventListener("visibilitychange", handleVisibility) + + // Set up interval for periodic pull + pullIntervalRef.current = setInterval(doPull, _BACKUP_PULL_INTERVAL_MS) + + return () => { + document.removeEventListener("visibilitychange", handleVisibility) + if (pullIntervalRef.current) clearInterval(pullIntervalRef.current) + } + }, [directoryName, space, spaceId]) + return null } From 4416f3140755da6ced2e6a00cbcad89153a9cdbc Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Fri, 13 Feb 2026 13:42:22 +0100 Subject: [PATCH 05/19] Add bidirectional sync toggle to BackupSettings UI --- src/lib/backup.tsx | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx index 86c0a61..0edbaa1 100644 --- a/src/lib/backup.tsx +++ b/src/lib/backup.tsx @@ -279,7 +279,15 @@ async function checkBackupPermission(): Promise { // Settings UI component function BackupSettings() { - let { enabled, directoryName, lastBackupAt, lastError } = useBackupStore() + let { + enabled, + bidirectional, + directoryName, + lastBackupAt, + lastPullAt, + lastError, + setBidirectional, + } = useBackupStore() let [isLoading, setIsLoading] = useState(false) if (!isBackupSupported()) return null @@ -307,6 +315,9 @@ function BackupSettings() { ? lastBackupDate.toLocaleString() : null + let lastPullDate = lastPullAt ? new Date(lastPullAt) : null + let formattedLastPull = lastPullDate ? lastPullDate.toLocaleString() : null + return (

@@ -317,22 +328,44 @@ function BackupSettings() { <>
- Backing up to folder + + {bidirectional ? "Syncing" : "Backing up"} to folder +

Folder: {directoryName}

{formattedLastBackup && ( -

+

Last backup: {formattedLastBackup}

)} + {bidirectional && formattedLastPull && ( +

+ Last sync: {formattedLastPull} +

+ )} {lastError && (
{lastError}
)} +
+ +

+ When enabled, changes made in the backup folder will be imported + into Alkalye. +

+
)}
-
@@ -738,12 +737,13 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { } }, [directoryName, space, spaceId, setDirectoryName]) - // Pull from filesystem (import changes) - always enabled for space backups + // Pull from filesystem (import changes) - only supported with FileSystem watch useEffect(() => { // Skip if no backup folder configured if (!directoryName) return // Skip if space not loaded if (!space?.$isLoaded || !space.documents?.$isLoaded) return + if (!supportsFileSystemWatch()) return // Requires watch API let docs = space.documents @@ -772,45 +772,31 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { } } - // Pull on mount and visibility change - doPull() - - let handleVisibility = () => { - if (document.visibilityState === "visible") { - doPull() - } - } - - document.addEventListener("visibilitychange", handleVisibility) - - // Use FileSystemDirectoryHandle.watch() for real-time file change detection (Chrome 110+) + // Set up watch for real-time file change detection let watchAborted = false async function setupWatch() { let handle = await getSpaceBackupHandle(spaceId) if (!handle) return - if (supportsFileSystemWatch()) { - let watcher = ( - handle as unknown as { - watch(options: { recursive: boolean }): { - addEventListener(event: string, callback: () => void): void - } + let watcher = ( + handle as unknown as { + watch(options: { recursive: boolean }): { + addEventListener(event: string, callback: () => void): void } - ).watch({ - recursive: true, - }) - watcher.addEventListener("change", () => { - if (!watchAborted) doPull() - }) - } + } + ).watch({ + recursive: true, + }) + watcher.addEventListener("change", () => { + if (!watchAborted) doPull() + }) } setupWatch() return () => { watchAborted = true - document.removeEventListener("visibilitychange", handleVisibility) } }, [directoryName, space, spaceId]) @@ -823,10 +809,6 @@ let spacesBackupQuery = { }, } as const satisfies ResolveQuery -function isBackupSupported(): boolean { - return "showDirectoryPicker" in window -} - async function getStoredHandle(): Promise { try { let handle = await idbGet(HANDLE_STORAGE_KEY) From 82f563501fa1da9633b8492c78ec9d96c89487d1 Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Sat, 14 Feb 2026 16:34:20 +0100 Subject: [PATCH 10/19] use FileSystemObserver, fix a few edgecases and add tests --- src/lib/backup-scenarios.test.ts | 571 +++++++++++++++++++++++ src/lib/backup-sync.test.ts | 410 +++++++++++++++-- src/lib/backup-sync.ts | 30 +- src/lib/backup.test.ts | 21 +- src/lib/backup.tsx | 767 ++++++++++++++++++++++--------- 5 files changed, 1534 insertions(+), 265 deletions(-) create mode 100644 src/lib/backup-scenarios.test.ts diff --git a/src/lib/backup-scenarios.test.ts b/src/lib/backup-scenarios.test.ts new file mode 100644 index 0000000..e11c767 --- /dev/null +++ b/src/lib/backup-scenarios.test.ts @@ -0,0 +1,571 @@ +import { beforeEach, describe, expect, it } from "vitest" +import { co, Group } from "jazz-tools" +import { createJazzTestAccount, setupJazzTestSync } from "jazz-tools/testing" +import { UserAccount, Document } from "@/schema" +import { getPath } from "@/editor/frontmatter" +import { getDocumentTitle } from "@/lib/document-utils" +import type { BackupDoc } from "./backup-sync" +import { readManifest } from "./backup-sync" +import { syncBackup, syncFromBackup } from "./backup" + +class MockWritableFileStream implements FileSystemWritableFileStream { + private stream = new WritableStream() + private saveContent: (content: string) => void + + constructor(saveContent: (content: string) => void) { + this.saveContent = saveContent + } + + get locked(): boolean { + return this.stream.locked + } + + abort(reason?: unknown): Promise { + return this.stream.abort(reason) + } + + close(): Promise { + return Promise.resolve() + } + + getWriter(): WritableStreamDefaultWriter { + return this.stream.getWriter() + } + + seek(_position: number): Promise { + return Promise.resolve() + } + + truncate(_size: number): Promise { + return Promise.resolve() + } + + write(data: string | Blob | ArrayBuffer): Promise + write(data: ArrayBufferView): Promise + write(data: { + type: "write" + data: string | Blob | ArrayBuffer | ArrayBufferView | null + }): Promise + write(data: { type: "seek"; position: number }): Promise + write(data: { type: "truncate"; size: number }): Promise + async write(data: unknown): Promise { + if (typeof data === "string") { + this.saveContent(data) + return + } + + if (data instanceof Blob) { + this.saveContent(await data.text()) + return + } + + if (data instanceof ArrayBuffer) { + this.saveContent(new TextDecoder().decode(new Uint8Array(data))) + return + } + + if (ArrayBuffer.isView(data)) { + let bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + + if (typeof data === "object" && data !== null && "type" in data) { + if ( + data.type === "write" && + "data" in data && + data.data !== undefined && + data.data !== null + ) { + let nestedData = data.data + if (typeof nestedData === "string") { + this.saveContent(nestedData) + return + } + if (nestedData instanceof Blob) { + this.saveContent(await nestedData.text()) + return + } + if (nestedData instanceof ArrayBuffer) { + this.saveContent(new TextDecoder().decode(new Uint8Array(nestedData))) + return + } + if (ArrayBuffer.isView(nestedData)) { + let bytes = new Uint8Array( + nestedData.buffer, + nestedData.byteOffset, + nestedData.byteLength, + ) + this.saveContent(new TextDecoder().decode(bytes)) + } + } + } + } + + get [Symbol.toStringTag](): string { + return "FileSystemWritableFileStream" + } +} + +class MockFileHandle implements FileSystemFileHandle { + kind = "file" as const + name: string + private getContent: () => string + private setContent: (content: string) => void + private getLastModified: () => number + private setLastModified: (lastModified: number) => void + + constructor( + name: string, + getContent: () => string, + setContent: (content: string) => void, + getLastModified: () => number, + setLastModified: (lastModified: number) => void, + ) { + this.name = name + this.getContent = getContent + this.setContent = setContent + this.getLastModified = getLastModified + this.setLastModified = setLastModified + } + + async getFile(): Promise { + return new File([this.getContent()], this.name, { + lastModified: this.getLastModified(), + }) + } + + async createWritable(): Promise { + return new MockWritableFileStream(content => { + this.setContent(content) + this.setLastModified(Date.now()) + }) + } + + isSameEntry(other: FileSystemHandle): Promise { + return Promise.resolve(other.kind === "file" && other.name === this.name) + } + + get [Symbol.toStringTag](): string { + return "FileSystemFileHandle" + } +} + +interface StoredFile { + content: string + lastModified: number +} + +function isDirectoryHandle( + handle: FileSystemHandle | undefined, +): handle is MockDirectoryHandle { + return handle?.kind === "directory" +} + +function isFileHandle( + handle: FileSystemHandle | undefined, +): handle is MockFileHandle { + return handle?.kind === "file" +} + +class MockDirectoryHandle implements FileSystemDirectoryHandle { + kind = "directory" as const + name: string + private children = new Map() + private files = new Map() + + constructor(name: string) { + this.name = name + } + + addFile(name: string, content: string, lastModified = Date.now()) { + this.files.set(name, { content, lastModified }) + let fileHandle = new MockFileHandle( + name, + () => this.files.get(name)?.content ?? "", + updatedContent => { + let entry = this.files.get(name) + this.files.set(name, { + content: updatedContent, + lastModified: entry?.lastModified ?? Date.now(), + }) + }, + () => this.files.get(name)?.lastModified ?? Date.now(), + lastModifiedValue => { + let entry = this.files.get(name) + this.files.set(name, { + content: entry?.content ?? "", + lastModified: lastModifiedValue, + }) + }, + ) + this.children.set(name, fileHandle) + } + + addDirectory(name: string, directory: MockDirectoryHandle) { + this.children.set(name, directory) + } + + entries(): AsyncIterableIterator<[string, FileSystemHandle]> { + let iter = this.children.entries() + return { + async next() { + let result = iter.next() + if (result.done) return { done: true, value: undefined } + return { done: false, value: result.value } + }, + [Symbol.asyncIterator]() { + return this + }, + } + } + + async getFileHandle( + name: string, + options?: { create?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!isFileHandle(handle)) { + if (options?.create) { + this.addFile(name, "") + let created = this.children.get(name) + if (!isFileHandle(created)) throw new Error("Failed creating file") + return created + } + throw new Error(`File not found: ${name}`) + } + return handle + } + + async getDirectoryHandle( + name: string, + options?: { create?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!isDirectoryHandle(handle)) { + if (options?.create) { + let created = new MockDirectoryHandle(name) + this.children.set(name, created) + return created + } + throw new Error(`Directory not found: ${name}`) + } + return handle + } + + async removeEntry( + name: string, + options?: { recursive?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!handle) return + + if (handle.kind === "directory" && !options?.recursive) { + throw new Error("Directory removal requires recursive flag") + } + + this.children.delete(name) + this.files.delete(name) + } + + resolve(): Promise { + return Promise.resolve([this.name]) + } + + queryPermission(): Promise<"granted"> { + return Promise.resolve("granted") + } + + requestPermission(): Promise<"granted"> { + return Promise.resolve("granted") + } + + isSameEntry(other: FileSystemHandle): Promise { + return Promise.resolve( + other.kind === "directory" && other.name === this.name, + ) + } + + get [Symbol.toStringTag](): string { + return "FileSystemDirectoryHandle" + } +} + +type LoadedAccount = co.loaded +type LoadedDoc = co.loaded + +describe("backup scenarios", () => { + let account: LoadedAccount + let docs: co.loaded>> + let root: MockDirectoryHandle + + beforeEach(async () => { + await setupJazzTestSync() + account = await createJazzTestAccount({ + isCurrentActiveAccount: true, + AccountSchema: UserAccount, + }) + + let loaded = await account.$jazz.ensureLoaded({ + resolve: { + root: { + documents: { + $each: { + content: true, + assets: { $each: { image: true, video: true } }, + }, + }, + }, + }, + }) + + docs = loaded.root.documents + root = new MockDirectoryHandle("root") + }) + + it("new doc in alkalye", async () => { + let doc = await createDoc(docs, "# New Doc\n\nHello") + await pushToBackup(root, docs) + + let manifest = await readManifest(root) + expect(manifest?.entries.some(entry => entry.docId === doc.$jazz.id)).toBe( + true, + ) + expect(await hasFile(root, "New Doc.md")).toBe(true) + }) + + it("new doc locally", async () => { + let initialCount = getLoadedDocs(docs).length + root.addFile("Local Note.md", "# Local Note\n\nFrom filesystem", 2_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.created).toBe(1) + + let loadedDocs = getLoadedDocs(docs) + expect(loadedDocs).toHaveLength(initialCount + 1) + let imported = loadedDocs.find(d => getDocumentTitle(d) === "Local Note") + expect(imported).toBeDefined() + }) + + it("renamed in alkalye", async () => { + let doc = await createDoc(docs, "# Hello World") + await pushToBackup(root, docs) + expect(await hasFile(root, "Hello World.md")).toBe(true) + + if (!doc.content?.$isLoaded) throw new Error("Doc content not loaded") + doc.content.$jazz.applyDiff("# Another Title") + doc.$jazz.set("updatedAt", new Date(Date.now() + 2_000)) + await pushToBackup(root, docs) + + expect(await hasFile(root, "Another Title.md")).toBe(true) + expect(await hasFile(root, "Hello World.md")).toBe(false) + }) + + it("renamed locally", async () => { + let doc = await createDoc(docs, "# Hello") + await pushToBackup(root, docs) + let countBeforePull = getLoadedDocs(docs).length + + let oldFile = await readFile(root, "Hello.md") + await removeFile(root, "Hello.md") + root.addFile("Renamed Locally.md", oldFile, 5_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.created).toBe(0) + + let loadedDocs = getLoadedDocs(docs) + expect(loadedDocs).toHaveLength(countBeforePull) + expect(loadedDocs.some(d => d.$jazz.id === doc.$jazz.id)).toBe(true) + }) + + it("renamed folder locally updates path in alkalye", async () => { + await createDoc(docs, "---\npath: work\n---\n\n# Folder Move") + await pushToBackup(root, docs) + + let source = await readFile(root, "work/Folder Move.md") + await removeFile(root, "work/Folder Move.md") + root.addFile("archive/Folder Move.md", source, 8_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find( + d => getDocumentTitle(d) === "Folder Move", + ) + expect(loaded).toBeDefined() + expect(getPath(loaded?.content?.toString() ?? "")).toBe("archive") + }) + + it("changed path in alkalye", async () => { + let doc = await createDoc(docs, "# Path Doc") + await pushToBackup(root, docs) + + if (!doc.content?.$isLoaded) throw new Error("Doc content not loaded") + doc.content.$jazz.applyDiff("---\npath: work/notes\n---\n\n# Path Doc") + doc.$jazz.set("updatedAt", new Date(Date.now() + 3_000)) + await pushToBackup(root, docs) + + expect(await hasFile(root, "work/notes/Path Doc.md")).toBe(true) + expect(await hasFile(root, "Path Doc.md")).toBe(false) + }) + + it("changed path locally is normalized to filesystem location", async () => { + await createDoc(docs, "---\npath: work\n---\n\n# Local Path") + await pushToBackup(root, docs) + + let source = "---\npath: notes\n---\n\n# Local Path" + root.addFile("work/Local Path.md", source, 7_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBeGreaterThanOrEqual(0) + + let loaded = getLoadedDocs(docs).find( + d => getDocumentTitle(d) === "Local Path", + ) + expect(loaded).toBeDefined() + expect(getPath(loaded?.content?.toString() ?? "")).toBe("work") + }) + + it("deleted in alkalye", async () => { + let doc = await createDoc(docs, "# Delete Me") + await pushToBackup(root, docs) + expect(await hasFile(root, "Delete Me.md")).toBe(true) + + doc.$jazz.set("deletedAt", new Date()) + doc.$jazz.set("updatedAt", new Date(Date.now() + 4_000)) + await pushToBackup(root, docs) + + expect(await hasFile(root, "Delete Me.md")).toBe(false) + let manifest = await readManifest(root) + expect(manifest?.entries.some(entry => entry.docId === doc.$jazz.id)).toBe( + false, + ) + }) + + it("deleted locally", async () => { + let doc = await createDoc(docs, "# Remove Local") + await pushToBackup(root, docs) + await removeFile(root, "Remove Local.md") + + let result = await syncFromBackup(root, docs, true) + expect(result.deleted).toBe(1) + + let target = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(target?.deletedAt).toBeTruthy() + }) + + it("edited both locally and in alkalye keeps document accessible and stable", async () => { + let doc = await createDoc(docs, "# Conflict\n\nbase") + await pushToBackup(root, docs) + + if (!doc.content?.$isLoaded) throw new Error("Doc content not loaded") + doc.content.$jazz.applyDiff("# Conflict\n\nbase\nfrom-alkalye") + doc.$jazz.set("updatedAt", new Date(Date.now() + 5_000)) + + let existing = await readFile(root, "Conflict.md") + root.addFile("Conflict.md", `${existing}\nfrom-local`, 9_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.errors).toHaveLength(0) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + expect(loaded?.deletedAt).toBeFalsy() + expect(loaded?.content?.toString()).toContain("Conflict") + }) +}) + +function getLoadedDocs( + docs: co.loaded>>, +): LoadedDoc[] { + let result: LoadedDoc[] = [] + for (let doc of docs) { + if (doc?.$isLoaded) result.push(doc) + } + return result +} + +async function createDoc( + docs: co.loaded>>, + content: string, +): Promise { + let group = Group.create() + let now = new Date() + let doc = Document.create( + { + version: 1, + content: co.plainText().create(content, group), + createdAt: now, + updatedAt: now, + }, + group, + ) + docs.$jazz.push(doc) + return doc +} + +async function pushToBackup( + handle: FileSystemDirectoryHandle, + docs: co.loaded>>, +): Promise { + let backupDocs: BackupDoc[] = [] + + for (let doc of docs) { + if (!doc?.$isLoaded || doc.deletedAt) continue + let content = doc.content?.toString() ?? "" + backupDocs.push({ + id: doc.$jazz.id, + title: getDocumentTitle(doc), + content, + path: getPath(content), + updatedAtMs: doc.updatedAt?.getTime() ?? 0, + assets: [], + }) + } + + await syncBackup(handle, backupDocs) +} + +async function hasFile( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + try { + await readFile(root, relativePath) + return true + } catch { + return false + } +} + +async function readFile( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + + let fileHandle = await dir.getFileHandle(parts[parts.length - 1]) + let file = await fileHandle.getFile() + return file.text() +} + +async function removeFile( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + + await dir.removeEntry(parts[parts.length - 1]) +} diff --git a/src/lib/backup-sync.test.ts b/src/lib/backup-sync.test.ts index e313f1c..839ed78 100644 --- a/src/lib/backup-sync.test.ts +++ b/src/lib/backup-sync.test.ts @@ -1,10 +1,14 @@ import { describe, it, expect } from "vitest" import { + computeDocLocations, + transformContentForBackup, + computeExpectedStructure, scanBackupFolder, transformContentForImport, readManifest, writeManifest, type BackupManifest, + type BackupDoc, } from "./backup-sync" // ============================================================================= @@ -19,14 +23,162 @@ function createMockBlob(content: string, type = "image/png"): Blob { return new Blob([content], { type }) } +class MockWritableFileStream implements FileSystemWritableFileStream { + private stream = new WritableStream() + private saveContent: (content: string) => void + + constructor(saveContent: (content: string) => void) { + this.saveContent = saveContent + } + + get locked(): boolean { + return this.stream.locked + } + + abort(reason?: unknown): Promise { + return this.stream.abort(reason) + } + + close(): Promise { + return Promise.resolve() + } + + getWriter(): WritableStreamDefaultWriter { + return this.stream.getWriter() + } + + seek(_position: number): Promise { + return Promise.resolve() + } + + truncate(_size: number): Promise { + return Promise.resolve() + } + + write(data: string | Blob | ArrayBuffer): Promise + write(data: ArrayBufferView): Promise + write(data: { + type: "write" + data: string | Blob | ArrayBuffer | ArrayBufferView | null + }): Promise + write(data: { type: "seek"; position: number }): Promise + write(data: { type: "truncate"; size: number }): Promise + async write(data: unknown): Promise { + if (typeof data === "string") { + this.saveContent(data) + return + } + + if (data instanceof Blob) { + this.saveContent(await data.text()) + return + } + + if (data instanceof ArrayBuffer) { + let bytes = new Uint8Array(data) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + + if (ArrayBuffer.isView(data)) { + let bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + + if (typeof data === "object" && data !== null && "type" in data) { + if ( + data.type === "write" && + "data" in data && + data.data !== undefined && + data.data !== null + ) { + let nestedData = data.data + if (typeof nestedData === "string") { + this.saveContent(nestedData) + return + } + if (nestedData instanceof Blob) { + this.saveContent(await nestedData.text()) + return + } + if (nestedData instanceof ArrayBuffer) { + let bytes = new Uint8Array(nestedData) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + if (ArrayBuffer.isView(nestedData)) { + let bytes = new Uint8Array( + nestedData.buffer, + nestedData.byteOffset, + nestedData.byteLength, + ) + this.saveContent(new TextDecoder().decode(bytes)) + } + } + } + } + + get [Symbol.toStringTag](): string { + return "FileSystemWritableFileStream" + } +} + +class MockFileHandle implements FileSystemFileHandle { + kind = "file" as const + name: string + private getContent: () => string | undefined + private setContent: (content: string) => void + private initialFile: File + + constructor( + name: string, + getContent: () => string | undefined, + setContent: (content: string) => void, + initialFile: File, + ) { + this.name = name + this.getContent = getContent + this.setContent = setContent + this.initialFile = initialFile + } + + async getFile(): Promise { + let content = this.getContent() + if (content !== undefined) return new File([content], this.name) + return this.initialFile + } + + async createWritable(): Promise { + return new MockWritableFileStream(this.setContent) + } + + isSameEntry(other: FileSystemHandle): Promise { + return Promise.resolve(other.kind === "file" && other.name === this.name) + } + + get [Symbol.toStringTag](): string { + return "FileSystemFileHandle" + } +} + +function isFileHandle( + handle: FileSystemHandle | undefined, +): handle is FileSystemFileHandle { + return handle?.kind === "file" +} + +function isDirectoryHandle( + handle: FileSystemHandle | undefined, +): handle is FileSystemDirectoryHandle { + return handle?.kind === "directory" +} + // Mock FileSystemDirectoryHandle for testing class MockDirectoryHandle implements FileSystemDirectoryHandle { kind = "directory" as const name: string - private children = new Map< - string, - FileSystemFileHandle | FileSystemDirectoryHandle - >() + private children = new Map() constructor(name: string) { this.name = name @@ -35,27 +187,12 @@ class MockDirectoryHandle implements FileSystemDirectoryHandle { private fileContents = new Map() addFile(name: string, file: File) { - let mockHandle: FileSystemFileHandle = { - kind: "file", + let mockHandle = new MockFileHandle( name, - getFile: async () => { - // Check if there's saved content from writeManifest - let content = this.fileContents.get(name) - if (content !== undefined) { - return new File([content], name) - } - return file - }, - createWritable: async () => { - return { - write: async (data: string | Blob) => { - let content = typeof data === "string" ? data : await data.text() - this.fileContents.set(name, content) - }, - close: async () => {}, - } - }, - } as unknown as FileSystemFileHandle + () => this.fileContents.get(name), + content => this.fileContents.set(name, content), + file, + ) this.children.set(name, mockHandle) } @@ -83,32 +220,20 @@ class MockDirectoryHandle implements FileSystemDirectoryHandle { options?: { create?: boolean }, ): Promise { let handle = this.children.get(name) - if (!handle || handle.kind !== "file") { + if (!isFileHandle(handle)) { if (options?.create) { - let mockHandle: FileSystemFileHandle = { - kind: "file", + let mockHandle = new MockFileHandle( name, - getFile: async () => { - let content = this.fileContents.get(name) - return new File([content ?? ""], name) - }, - createWritable: async () => { - return { - write: async (data: string | Blob) => { - let content = - typeof data === "string" ? data : await data.text() - this.fileContents.set(name, content) - }, - close: async () => {}, - } - }, - } as unknown as FileSystemFileHandle + () => this.fileContents.get(name), + content => this.fileContents.set(name, content), + new File([""], name), + ) this.children.set(name, mockHandle) return mockHandle } throw new Error(`File not found: ${name}`) } - return handle as FileSystemFileHandle + return handle } async getDirectoryHandle( @@ -116,7 +241,7 @@ class MockDirectoryHandle implements FileSystemDirectoryHandle { options?: { create?: boolean }, ): Promise { let handle = this.children.get(name) - if (!handle || handle.kind !== "directory") { + if (!isDirectoryHandle(handle)) { if (options?.create) { let newDir = new MockDirectoryHandle(name) this.children.set(name, newDir) @@ -124,7 +249,7 @@ class MockDirectoryHandle implements FileSystemDirectoryHandle { } throw new Error(`Directory not found: ${name}`) } - return handle as FileSystemDirectoryHandle + return handle } removeEntry(): Promise { @@ -152,6 +277,171 @@ class MockDirectoryHandle implements FileSystemDirectoryHandle { } } +// ============================================================================= +// Compute Doc Locations +// ============================================================================= + +describe("computeDocLocations", () => { + function createDoc(input: { + id: string + title: string + path: string | null + assets?: { id: string; name: string; blob: Blob }[] + }): BackupDoc { + return { + id: input.id, + title: input.title, + content: "# Content", + path: input.path, + updatedAtMs: 0, + assets: input.assets ?? [], + } + } + + it("uses title.md for root docs without assets", () => { + let docs = [createDoc({ id: "d1", title: "Hello World", path: null })] + let locations = computeDocLocations(docs) + let loc = locations.get("d1") + + expect(loc?.dirPath).toBe("") + expect(loc?.filename).toBe("Hello World.md") + expect(loc?.hasOwnFolder).toBe(false) + }) + + it("creates doc folder for docs with assets", () => { + let docs = [ + createDoc({ + id: "d1", + title: "Project Note", + path: null, + assets: [{ id: "a1", name: "Image", blob: createMockBlob("img") }], + }), + ] + let locations = computeDocLocations(docs) + let loc = locations.get("d1") + + expect(loc?.dirPath).toBe("Project Note") + expect(loc?.filename).toBe("Project Note.md") + expect(loc?.hasOwnFolder).toBe(true) + }) + + it("disambiguates title collisions in same folder", () => { + let docs = [ + createDoc({ id: "doc-11111111", title: "Same", path: "work" }), + createDoc({ id: "doc-22222222", title: "Same", path: "work" }), + ] + let locations = computeDocLocations(docs) + let first = locations.get("doc-11111111") + let second = locations.get("doc-22222222") + + expect(first?.filename).toBe("Same.md") + expect(second?.filename).toContain("Same") + expect(second?.filename).toContain("22222222") + }) + + it("does not disambiguate same title in different folders", () => { + let docs = [ + createDoc({ id: "d1", title: "Same", path: "work" }), + createDoc({ id: "d2", title: "Same", path: "personal" }), + ] + let locations = computeDocLocations(docs) + + expect(locations.get("d1")?.filename).toBe("Same.md") + expect(locations.get("d2")?.filename).toBe("Same.md") + }) + + it("disambiguates duplicate asset filenames", () => { + let docs = [ + createDoc({ + id: "d1", + title: "Assets", + path: null, + assets: [ + { id: "a1", name: "shot", blob: createMockBlob("one") }, + { id: "a2", name: "shot", blob: createMockBlob("two") }, + ], + }), + ] + let locations = computeDocLocations(docs) + let loc = locations.get("d1") + + expect(loc?.assetFiles.get("a1")).toBeDefined() + expect(loc?.assetFiles.get("a2")).toBeDefined() + expect(loc?.assetFiles.get("a1")).not.toBe(loc?.assetFiles.get("a2")) + }) +}) + +// ============================================================================= +// Transform Content for Backup +// ============================================================================= + +describe("transformContentForBackup", () => { + it("transforms asset: references to assets paths", () => { + let assetFiles = new Map([ + ["asset1", "photo.png"], + ["asset2", "clip.jpg"], + ]) + let content = "![A](asset:asset1)\n![B](asset:asset2)" + + let result = transformContentForBackup(content, assetFiles) + + expect(result).toContain("![A](assets/photo.png)") + expect(result).toContain("![B](assets/clip.jpg)") + }) + + it("keeps unmatched asset references unchanged", () => { + let assetFiles = new Map([["asset1", "photo.png"]]) + let content = "![A](asset:missing)" + + let result = transformContentForBackup(content, assetFiles) + + expect(result).toBe("![A](asset:missing)") + }) +}) + +// ============================================================================= +// Compute Expected Structure +// ============================================================================= + +describe("computeExpectedStructure", () => { + it("includes parent directories and markdown files", () => { + let docs: BackupDoc[] = [ + { + id: "d1", + title: "Note", + content: "x", + path: "work/notes", + updatedAtMs: 0, + assets: [], + }, + ] + let locations = computeDocLocations(docs) + let expected = computeExpectedStructure(docs, locations) + + expect(expected.expectedPaths.has("work")).toBe(true) + expect(expected.expectedPaths.has("work/notes")).toBe(true) + expect(expected.expectedFiles.get("work/notes")?.has("Note.md")).toBe(true) + }) + + it("includes assets folder for docs with assets", () => { + let docs: BackupDoc[] = [ + { + id: "d1", + title: "Note", + content: "x", + path: "work", + updatedAtMs: 0, + assets: [{ id: "a1", name: "image", blob: createMockBlob("img") }], + }, + ] + let locations = computeDocLocations(docs) + let expected = computeExpectedStructure(docs, locations) + + expect(expected.expectedPaths.has("work/Note")).toBe(true) + expect(expected.expectedPaths.has("work/Note/assets")).toBe(true) + }) +}) + // ============================================================================= // Transform Content for Import // ============================================================================= @@ -258,6 +548,36 @@ describe("readManifest", () => { expect(result).toBeNull() }) + + it("returns null for invalid entries shape", async () => { + let root = new MockDirectoryHandle("root") + let invalidManifest = { + version: 1, + entries: [{ docId: "d1" }], + lastSyncAt: new Date().toISOString(), + } + + root.addFile( + ".alkalye-manifest.json", + new File([JSON.stringify(invalidManifest)], ".alkalye-manifest.json"), + ) + + let result = await readManifest(root) + + expect(result).toBeNull() + }) + + it("returns null for malformed JSON", async () => { + let root = new MockDirectoryHandle("root") + root.addFile( + ".alkalye-manifest.json", + new File(["{invalid"], ".alkalye-manifest.json"), + ) + + let result = await readManifest(root) + + expect(result).toBeNull() + }) }) describe("writeManifest", () => { diff --git a/src/lib/backup-sync.ts b/src/lib/backup-sync.ts index f5859b1..ad17394 100644 --- a/src/lib/backup-sync.ts +++ b/src/lib/backup-sync.ts @@ -1,4 +1,5 @@ import { getExtensionFromBlob, sanitizeFilename } from "@/lib/export" +import { z } from "zod" export { computeDocLocations, @@ -21,6 +22,7 @@ interface BackupDoc { title: string content: string path: string | null + updatedAtMs: number assets: { id: string; name: string; blob: Blob }[] } @@ -39,6 +41,7 @@ interface ExpectedStructure { interface ManifestEntry { docId: string relativePath: string + locationKey?: string contentHash: string lastSyncedAt: string assets: { name: string; hash: string }[] @@ -58,6 +61,26 @@ interface ScannedFile { lastModified: number } +let manifestAssetSchema = z.object({ + name: z.string(), + hash: z.string(), +}) + +let manifestEntrySchema = z.object({ + docId: z.string(), + relativePath: z.string(), + locationKey: z.string().optional(), + contentHash: z.string(), + lastSyncedAt: z.string(), + assets: z.array(manifestAssetSchema), +}) + +let backupManifestSchema = z.object({ + version: z.literal(1), + entries: z.array(manifestEntrySchema), + lastSyncAt: z.string(), +}) + function computeDocLocations(docs: BackupDoc[]): Map { let docLocations = new Map() let usedNames = new Map>() // parentPath -> used names (lowercase) @@ -249,10 +272,9 @@ async function readManifest( let file = await fileHandle.getFile() let text = await file.text() let parsed = JSON.parse(text) - if (parsed.version === 1) { - return parsed as BackupManifest - } - return null + let validated = backupManifestSchema.safeParse(parsed) + if (!validated.success) return null + return validated.data } catch { return null } diff --git a/src/lib/backup.test.ts b/src/lib/backup.test.ts index 7673856..661e4f7 100644 --- a/src/lib/backup.test.ts +++ b/src/lib/backup.test.ts @@ -1,9 +1,5 @@ import { describe, it, expect } from "vitest" -import { hashContent } from "./backup" - -// ============================================================================= -// Hash Content -// ============================================================================= +import { hashContent, syncFromBackup } from "./backup" describe("hashContent", () => { it("returns consistent hash for same content", async () => { @@ -36,14 +32,21 @@ describe("hashContent", () => { expect(hash1).toBe(hash2) }) -}) -// ============================================================================= -// Bidirectional Sync (Exported for Testing) -// ============================================================================= + it("changes hash when only one character changes", async () => { + let hash1 = await hashContent("abc") + let hash2 = await hashContent("abd") + + expect(hash1).not.toBe(hash2) + }) +}) describe("bidirectional sync exports", () => { it("exports hashContent for testing", () => { expect(typeof hashContent).toBe("function") }) + + it("exports syncFromBackup for integration tests", () => { + expect(typeof syncFromBackup).toBe("function") + }) }) diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx index b40a9e0..4ced114 100644 --- a/src/lib/backup.tsx +++ b/src/lib/backup.tsx @@ -14,7 +14,7 @@ import { VideoAsset, } from "@/schema" import { getDocumentTitle } from "@/lib/document-utils" -import { getPath } from "@/editor/frontmatter" +import { getPath, parseFrontmatter } from "@/editor/frontmatter" import { Button } from "@/components/ui/button" import { get as idbGet, set as idbSet, del as idbDel } from "idb-keyval" import { @@ -23,6 +23,7 @@ import { computeExpectedStructure, scanBackupFolder, readManifest, + writeManifest, transformContentForImport, type BackupDoc, type ScannedFile, @@ -43,13 +44,27 @@ export { checkBackupPermission, // Exported for testing hashContent, + syncBackup, syncFromBackup, type ScannedFile, } // File System Access API type augmentation declare global { + interface FileSystemObserver { + observe( + handle: FileSystemDirectoryHandle, + options?: { recursive?: boolean }, + ): Promise + disconnect(): void + } + interface Window { + FileSystemObserver?: { + new ( + onChange: (records: unknown[], observer: FileSystemObserver) => void, + ): FileSystemObserver + } showDirectoryPicker(options?: { mode?: "read" | "readwrite" }): Promise @@ -65,19 +80,13 @@ declare global { } } -let BACKUP_DEBOUNCE_MS = 5000 - -function supportsFileSystemWatch(): boolean { - if (typeof FileSystemDirectoryHandle === "undefined") return false - let proto = Object.getPrototypeOf(FileSystemDirectoryHandle.prototype) - return "watch" in proto -} - -function isBackupSupported(): boolean { - return "showDirectoryPicker" in window -} +let BACKUP_DEBOUNCE_MS = 1200 let HANDLE_STORAGE_KEY = "backup-directory-handle" +let preferredRelativePathByDocId = new Map() +let recentImportedRelativePaths = new Map() +let RECENT_IMPORT_WINDOW_MS = 30_000 +let spaceLastPullAtById = new Map() interface BackupState { enabled: boolean @@ -132,6 +141,13 @@ type LoadedDocument = co.loaded< type DocumentList = co.loaded>> type Account = co.loaded +interface SyncFromBackupResult { + created: number + updated: number + deleted: number + errors: string[] +} + let backupQuery = { root: { documents: { @@ -145,6 +161,7 @@ function BackupSubscriber() { let { enabled, bidirectional, + lastPullAt, setLastBackupAt, setLastPullAt, setLastError, @@ -154,6 +171,8 @@ function BackupSubscriber() { let me = useAccount(UserAccount, { resolve: backupQuery }) let debounceRef = useRef | null>(null) let lastContentHashRef = useRef("") + let isPushingRef = useRef(false) + let isPullingRef = useRef(false) // Push to filesystem (backup) useEffect(() => { @@ -188,12 +207,15 @@ function BackupSubscriber() { (d): d is LoadedDocument => d?.$isLoaded === true, ) let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) + isPushingRef.current = true await syncBackup(handle, backupDocs) setLastBackupAt(new Date().toISOString()) setLastError(null) } catch (e) { setLastError(e instanceof Error ? e.message : "Backup failed") + } finally { + isPushingRef.current = false } }, BACKUP_DEBOUNCE_MS) @@ -202,21 +224,29 @@ function BackupSubscriber() { } }, [enabled, me, setLastBackupAt, setLastError, setEnabled, setDirectoryName]) - // Pull from filesystem (import changes) - only supported with FileSystem watch + // Pull from filesystem (import changes) - only supported with FileSystemObserver useEffect(() => { if (!enabled || !bidirectional || !me.$isLoaded) return - if (!supportsFileSystemWatch()) return // Requires watch API + if (!supportsFileSystemWatch()) return let docs = me.root?.documents if (!docs?.$isLoaded) return async function doPull() { try { + if (isPushingRef.current) return + if (isPullingRef.current) return + isPullingRef.current = true let handle = await getBackupHandle() if (!handle) return - if (!docs.$isLoaded) return - let result = await syncFromBackup(handle, docs as DocumentList, true) + if (!docs.$isLoaded || !isDocumentList(docs)) return + let result = await syncFromBackup( + handle, + docs, + true, + toTimestamp(lastPullAt), + ) if (result.errors.length > 0) { console.warn("Backup pull errors:", result.errors) } @@ -224,36 +254,36 @@ function BackupSubscriber() { setLastPullAt(new Date().toISOString()) } catch (e) { console.error("Backup pull failed:", e) + } finally { + isPullingRef.current = false } } - // Set up watch for real-time file change detection + // Set up observer for real-time file change detection let watchAborted = false + let stopWatching: (() => void) | null = null async function setupWatch() { let handle = await getBackupHandle() if (!handle) return - let watcher = ( - handle as unknown as { - watch(options: { recursive: boolean }): { - addEventListener(event: string, callback: () => void): void - } - } - ).watch({ - recursive: true, - }) - watcher.addEventListener("change", () => { + let stop = await observeDirectoryChanges(handle, () => { if (!watchAborted) doPull() }) + if (watchAborted) { + stop?.() + return + } + stopWatching = stop } setupWatch() return () => { watchAborted = true + stopWatching?.() } - }, [enabled, bidirectional, me, setLastPullAt]) + }, [enabled, bidirectional, me, setLastPullAt, lastPullAt]) return null } @@ -416,7 +446,7 @@ function BackupSettings() {

{supportsFileSystemWatch() ? "When enabled, changes made in the backup folder will be imported into Alkalye." - : "Requires a Chromium-based browser with File System Watch support."} + : "Requires a Chromium-based browser with File System Observer support."}

@@ -475,7 +505,8 @@ function getSpaceBackupPath(spaceId: string): string | null { let key = getSpaceBackupStorageKey(spaceId) let stored = localStorage.getItem(key) if (!stored) return null - let parsed = JSON.parse(stored) as SpaceBackupState + let parsed = JSON.parse(stored) + if (!isSpaceBackupState(parsed)) return null return parsed.directoryName } catch { return null @@ -665,13 +696,148 @@ function SpacesBackupSubscriber() { ) } +// Exported for testing +async function hashContent(content: string): Promise { + // Simple hash using built-in crypto + let encoder = new TextEncoder() + let data = encoder.encode(content) + let hashBuffer = await crypto.subtle.digest("SHA-256", data) + let hashArray = Array.from(new Uint8Array(hashBuffer)) + return hashArray + .map(b => b.toString(16).padStart(2, "0")) + .join("") + .slice(0, 16) +} + +async function syncBackup( + handle: FileSystemDirectoryHandle, + docs: BackupDoc[], +): Promise { + await performSyncBackup(handle, docs) +} + +async function syncFromBackup( + handle: FileSystemDirectoryHandle, + targetDocs: DocumentList, + canWrite: boolean, + lastPullAtMs: number | null = null, +): Promise { + let result: SyncFromBackupResult = { + created: 0, + updated: 0, + deleted: 0, + errors: [], + } + + let manifest = await readManifest(handle) + let scannedFiles = await scanBackupFolder(handle) + let listOwner = targetDocs.$jazz.owner + + // Build maps for lookup + let manifestByPath = new Map( + manifest?.entries.map(e => [e.relativePath, e]) ?? [], + ) + let scannedByPath = new Map(scannedFiles.map(f => [f.relativePath, f])) + let matchedManifestDocIds = new Set() + + // Process new and updated files + for (let file of scannedFiles) { + try { + let contentHash = await hashContent(file.content) + let manifestEntry = manifestByPath.get(file.relativePath) + if (manifestEntry) { + matchedManifestDocIds.add(manifestEntry.docId) + if (lastPullAtMs !== null && file.lastModified <= lastPullAtMs) { + continue + } + } + + if (!manifestEntry) { + let movedEntry = findMovedManifestEntry( + manifest, + scannedByPath, + matchedManifestDocIds, + contentHash, + ) + if (movedEntry) { + manifestEntry = movedEntry + matchedManifestDocIds.add(movedEntry.docId) + } + } + + if (!manifestEntry) { + // New file - create document + if (!canWrite) { + result.errors.push(`Cannot create ${file.name}: no write permission`) + continue + } + if (wasRecentlyImported(file.relativePath)) continue + let newDocId = await createDocFromFile(file, targetDocs, listOwner) + preferredRelativePathByDocId.set(newDocId, file.relativePath) + markRecentlyImported(file.relativePath) + result.created++ + } else if ( + manifestEntry.contentHash !== contentHash || + manifestEntry.relativePath !== file.relativePath + ) { + preferredRelativePathByDocId.set(manifestEntry.docId, file.relativePath) + // File changed or moved - update document + if (!canWrite) { + result.errors.push(`Cannot update ${file.name}: no write permission`) + continue + } + let didUpdate = await updateDocFromFile( + file, + manifestEntry.docId, + targetDocs, + ) + if (didUpdate) { + result.updated++ + } else { + result.errors.push( + `Skipped update for ${file.relativePath}: target document not loaded`, + ) + } + } + } catch (err) { + result.errors.push( + `Failed to process ${file.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, + ) + } + } + + // Handle deletions (files in manifest but not on disk) + if (manifest && canWrite) { + for (let entry of manifest.entries) { + if (matchedManifestDocIds.has(entry.docId)) continue + if (!scannedByPath.has(entry.relativePath)) { + try { + let doc = targetDocs.find(d => d?.$jazz.id === entry.docId) + if (doc?.$isLoaded && !doc.deletedAt) { + // Soft delete + doc.$jazz.set("deletedAt", new Date()) + doc.$jazz.set("updatedAt", new Date()) + result.deleted++ + } + } catch (err) { + result.errors.push( + `Failed to delete ${entry.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, + ) + } + } + } + } + + return result +} + // ============================================================================= // Helper functions (used by exported functions above) // ============================================================================= // Space backup subscriber - handles backup sync for a single space -let SPACE_BACKUP_DEBOUNCE_MS = 5000 +let SPACE_BACKUP_DEBOUNCE_MS = 1200 interface SpaceBackupSubscriberProps { spaceId: string @@ -681,9 +847,11 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) let debounceRef = useRef | null>(null) let lastContentHashRef = useRef("") + let isPushingRef = useRef(false) + let isPullingRef = useRef(false) // Load space with documents - let space = useCoState(Space, spaceId as Parameters[1], { + let space = useCoState(Space, spaceId, { resolve: { documents: { $each: { content: true, assets: { $each: { image: true } } }, @@ -726,9 +894,12 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { (d): d is LoadedDocument => d?.$isLoaded === true, ) let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) + isPushingRef.current = true await syncBackup(handle, backupDocs) } catch (e) { console.error(`Space backup failed for ${spaceId}:`, e) + } finally { + isPushingRef.current = false } }, SPACE_BACKUP_DEBOUNCE_MS) @@ -737,13 +908,13 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { } }, [directoryName, space, spaceId, setDirectoryName]) - // Pull from filesystem (import changes) - only supported with FileSystem watch + // Pull from filesystem (import changes) - only supported with FileSystemObserver useEffect(() => { // Skip if no backup folder configured if (!directoryName) return // Skip if space not loaded if (!space?.$isLoaded || !space.documents?.$isLoaded) return - if (!supportsFileSystemWatch()) return // Requires watch API + if (!supportsFileSystemWatch()) return let docs = space.documents @@ -755,48 +926,53 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { async function doPull() { try { + if (isPushingRef.current) return + if (isPullingRef.current) return + isPullingRef.current = true let handle = await getSpaceBackupHandle(spaceId) if (!handle) return - if (!docs.$isLoaded) return + if (!docs.$isLoaded || !isDocumentList(docs)) return let result = await syncFromBackup( handle, - docs as DocumentList, + docs, canWrite, + spaceLastPullAtById.get(spaceId) ?? null, ) if (result.errors.length > 0) { console.warn(`Space ${spaceId} pull errors:`, result.errors) } + spaceLastPullAtById.set(spaceId, Date.now()) } catch (e) { console.error(`Space backup pull failed for ${spaceId}:`, e) + } finally { + isPullingRef.current = false } } - // Set up watch for real-time file change detection + // Set up observer for real-time file change detection let watchAborted = false + let stopWatching: (() => void) | null = null async function setupWatch() { let handle = await getSpaceBackupHandle(spaceId) if (!handle) return - let watcher = ( - handle as unknown as { - watch(options: { recursive: boolean }): { - addEventListener(event: string, callback: () => void): void - } - } - ).watch({ - recursive: true, - }) - watcher.addEventListener("change", () => { + let stop = await observeDirectoryChanges(handle, () => { if (!watchAborted) doPull() }) + if (watchAborted) { + stop?.() + return + } + stopWatching = stop } setupWatch() return () => { watchAborted = true + stopWatching?.() } }, [directoryName, space, spaceId]) @@ -829,7 +1005,7 @@ async function clearHandle(): Promise { async function verifyPermission( handle: FileSystemDirectoryHandle, ): Promise { - let opts = { mode: "readwrite" } as const + let opts: { mode: "readwrite" } = { mode: "readwrite" } if ((await handle.queryPermission(opts)) === "granted") return true if ((await handle.requestPermission(opts)) === "granted") return true return false @@ -858,6 +1034,7 @@ async function prepareBackupDoc(doc: LoadedDocument): Promise { let content = doc.content?.toString() ?? "" let title = getDocumentTitle(doc) let path = getPath(content) + let updatedAtMs = doc.updatedAt?.getTime() ?? 0 let assets: BackupDoc["assets"] = [] if (doc.assets?.$isLoaded) { @@ -880,7 +1057,7 @@ async function prepareBackupDoc(doc: LoadedDocument): Promise { } } - return { id: doc.$jazz.id, title, content, path, assets } + return { id: doc.$jazz.id, title, content, path, updatedAtMs, assets } } async function getOrCreateDirectory( @@ -935,34 +1112,118 @@ async function listDirectories( return dirs } -async function syncBackup( +async function performSyncBackup( handle: FileSystemDirectoryHandle, docs: BackupDoc[], ): Promise { let docLocations = computeDocLocations(docs) - - // Write all documents and their assets + let existingManifest = await readManifest(handle) + let existingEntriesByDocId = new Map( + existingManifest?.entries.map(entry => [entry.docId, entry]) ?? [], + ) + let manifestEntries: { + docId: string + relativePath: string + locationKey: string + contentHash: string + lastSyncedAt: string + assets: { name: string; hash: string }[] + }[] = [] + let hasFilesystemChanges = false + let nowIso = new Date().toISOString() + + // Write only changed documents and assets for (let doc of docs) { let loc = docLocations.get(doc.id)! + let locationKey = getDocLocationKey(doc) + let computedRelativePath = loc.dirPath + ? `${loc.dirPath}/${loc.filename}` + : loc.filename + let existingEntry = existingEntriesByDocId.get(doc.id) + let preferredRelativePath = preferredRelativePathByDocId.get(doc.id) + let finalRelativePath = computedRelativePath + if (existingEntry) { + if (existingEntry.locationKey === locationKey) { + finalRelativePath = existingEntry.relativePath + } + } + if (preferredRelativePath) { + finalRelativePath = preferredRelativePath + } + let finalLocation = buildLocationFromRelativePath(loc, finalRelativePath) + docLocations.set(doc.id, finalLocation) + loc = finalLocation + let dir = loc.dirPath ? await getOrCreateDirectory(handle, loc.dirPath) : handle let exportedContent = transformContentForBackup(doc.content, loc.assetFiles) - await writeFile(dir, loc.filename, exportedContent) - - // Write assets if any - if (doc.assets.length > 0) { - let assetsDir = await dir.getDirectoryHandle("assets", { create: true }) - for (let asset of doc.assets) { - let filename = loc.assetFiles.get(asset.id)! - await writeFile(assetsDir, filename, asset.blob) + let contentHash = await hashContent(exportedContent) + let relativePath = finalRelativePath + let assets: { name: string; hash: string }[] = [] + for (let asset of doc.assets) { + let filename = loc.assetFiles.get(asset.id)! + assets.push({ + name: filename, + hash: await hashBlob(asset.blob), + }) + } + + let shouldWriteDoc = + !existingEntry || + existingEntry.relativePath !== relativePath || + existingEntry.contentHash !== contentHash || + !areManifestAssetsEqual(existingEntry.assets, assets) + + if (shouldWriteDoc) { + hasFilesystemChanges = true + await writeFile(dir, loc.filename, exportedContent) + + // Write assets if any + if (doc.assets.length > 0) { + let assetsDir = await dir.getDirectoryHandle("assets", { create: true }) + for (let asset of doc.assets) { + let filename = loc.assetFiles.get(asset.id)! + await writeFile(assetsDir, filename, asset.blob) + } } } + + manifestEntries.push({ + docId: doc.id, + relativePath, + locationKey, + contentHash, + lastSyncedAt: shouldWriteDoc + ? nowIso + : (existingEntry?.lastSyncedAt ?? nowIso), + assets, + }) } - // Clean up orphaned files and directories - await cleanupOrphanedFiles(handle, docs, docLocations) + let docsChanged = + existingEntriesByDocId.size !== manifestEntries.length || + hasFilesystemChanges + + if (docsChanged) { + // Clean up orphaned files and directories + await cleanupOrphanedFiles(handle, docs, docLocations) + + await writeManifest(handle, { + version: 1, + entries: manifestEntries, + lastSyncAt: nowIso, + }) + + for (let entry of manifestEntries) { + recentImportedRelativePaths.delete(entry.relativePath) + } + } + + for (let doc of docs) { + preferredRelativePathByDocId.delete(doc.id) + } } async function cleanupOrphanedFiles( @@ -1029,155 +1290,23 @@ async function cleanupOrphanedFiles( await cleanDir(handle, "") } -function getSpaceBackupStorageKey(spaceId: string): string { - return `${SPACE_BACKUP_KEY_PREFIX}${spaceId}` -} - -async function getSpaceBackupHandle( - spaceId: string, -): Promise { - try { - let handle = await idbGet( - `${HANDLE_STORAGE_KEY}-space-${spaceId}`, - ) - if (!handle) return null - let hasPermission = await verifyPermission(handle) - if (!hasPermission) return null - return handle - } catch { - return null - } -} - -function getSpacesWithBackup( - me: ReturnType< - typeof useAccount - >, - _storageVersion: number, -): string[] { - if (!me.$isLoaded || !me.root?.spaces?.$isLoaded) return [] - - let spaceIds: string[] = [] - for (let space of Array.from(me.root.spaces)) { - if (!space?.$isLoaded) continue - let backupPath = getSpaceBackupPath(space.$jazz.id) - if (backupPath) { - spaceIds.push(space.$jazz.id) - } - } - return spaceIds -} - -// ============================================================================= -// Bidirectional Sync - Pull from filesystem -// ============================================================================= - -// Exported for testing -async function hashContent(content: string): Promise { - // Simple hash using built-in crypto - let encoder = new TextEncoder() - let data = encoder.encode(content) - let hashBuffer = await crypto.subtle.digest("SHA-256", data) - let hashArray = Array.from(new Uint8Array(hashBuffer)) - return hashArray - .map(b => b.toString(16).padStart(2, "0")) - .join("") - .slice(0, 16) -} - -async function syncFromBackup( - handle: FileSystemDirectoryHandle, - targetDocs: DocumentList, - canWrite: boolean, -): Promise<{ - created: number - updated: number - deleted: number - errors: string[] -}> { - let result = { created: 0, updated: 0, deleted: 0, errors: [] as string[] } - - let manifest = await readManifest(handle) - let scannedFiles = await scanBackupFolder(handle) - let listOwner = targetDocs.$jazz.owner - - // Build maps for lookup - let manifestByPath = new Map( - manifest?.entries.map(e => [e.relativePath, e]) ?? [], - ) - let scannedByPath = new Map(scannedFiles.map(f => [f.relativePath, f])) - - // Process new and updated files - for (let file of scannedFiles) { - try { - let contentHash = await hashContent(file.content) - let manifestEntry = manifestByPath.get(file.relativePath) - - if (!manifestEntry) { - // New file - create document - if (!canWrite) { - result.errors.push(`Cannot create ${file.name}: no write permission`) - continue - } - await createDocFromFile(file, targetDocs, listOwner) - result.created++ - } else if (manifestEntry.contentHash !== contentHash) { - // File changed - update document - if (!canWrite) { - result.errors.push(`Cannot update ${file.name}: no write permission`) - continue - } - await updateDocFromFile( - file, - manifestEntry.docId, - targetDocs, - listOwner, - ) - result.updated++ - } - } catch (err) { - result.errors.push( - `Failed to process ${file.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, - ) - } - } - - // Handle deletions (files in manifest but not on disk) - if (manifest && canWrite) { - for (let entry of manifest.entries) { - if (!scannedByPath.has(entry.relativePath)) { - try { - let doc = targetDocs.find(d => d?.$jazz.id === entry.docId) - if (doc?.$isLoaded && !doc.deletedAt) { - // Soft delete - doc.$jazz.set("deletedAt", new Date()) - doc.$jazz.set("updatedAt", new Date()) - result.deleted++ - } - } catch (err) { - result.errors.push( - `Failed to delete ${entry.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, - ) - } - } - } - } - - return result -} - async function createDocFromFile( file: ScannedFile, targetDocs: DocumentList, listOwner: Group | Account, -): Promise { +): Promise { // Transform asset references back to asset: format let assetFiles = new Map() for (let asset of file.assets) { let id = crypto.randomUUID() assetFiles.set(id, asset.name) } - let content = transformContentForImport(file.content, assetFiles) + let transformedContent = transformContentForImport(file.content, assetFiles) + let content = applyPathFromRelativePath( + transformedContent, + file.relativePath, + file.assets.length > 0, + ) // Create doc-specific group with list owner as parent let docGroup = Group.create() @@ -1244,21 +1373,19 @@ async function createDocFromFile( ) targetDocs.$jazz.push(newDoc) + return newDoc.$jazz.id } async function updateDocFromFile( file: ScannedFile, docId: string, targetDocs: DocumentList, - listOwner: Group | Account, -): Promise { +): Promise { let doc = targetDocs.find( (d): d is LoadedDocument => d?.$isLoaded === true && d.$jazz.id === docId, ) if (!doc || !doc.content?.$isLoaded) { - // Doc doesn't exist or content not loaded, treat as create - await createDocFromFile(file, targetDocs, listOwner) - return + return false } // Update content @@ -1267,11 +1394,237 @@ async function updateDocFromFile( let id = crypto.randomUUID() assetFiles.set(id, asset.name) } - let content = transformContentForImport(file.content, assetFiles) + let transformedContent = transformContentForImport(file.content, assetFiles) + let content = applyPathFromRelativePath( + transformedContent, + file.relativePath, + file.assets.length > 0, + ) doc.content.$jazz.applyDiff(content) doc.$jazz.set("updatedAt", new Date()) // TODO: Handle asset updates (add/remove/replace assets) // For now, we just update the content. Asset changes require more complex logic. + return true +} + +function isDocumentList(value: unknown): value is DocumentList { + if (typeof value !== "object" || value === null) return false + return "$jazz" in value && "find" in value +} + +function isSpaceBackupState(value: unknown): value is SpaceBackupState { + if (typeof value !== "object" || value === null) return false + if (!("directoryName" in value)) return false + return value.directoryName === null || typeof value.directoryName === "string" +} + +function findMovedManifestEntry( + manifest: Awaited>, + scannedByPath: Map, + matchedManifestDocIds: Set, + contentHash: string, +) { + if (!manifest) return null + + for (let entry of manifest.entries) { + if (matchedManifestDocIds.has(entry.docId)) continue + if (scannedByPath.has(entry.relativePath)) continue + if (entry.contentHash === contentHash) return entry + } + + return null +} + +function applyPathFromRelativePath( + content: string, + relativePath: string, + hasAssets: boolean, +): string { + let diskPath = derivePathFromRelativePath(relativePath, hasAssets) + let { frontmatter } = parseFrontmatter(content) + let currentPath = getPath(content) + + if (!frontmatter) { + if (!diskPath) return content + return `---\npath: ${diskPath}\n---\n\n${content}` + } + + if (currentPath === diskPath) return content + + if (currentPath && !diskPath) { + return content.replace( + /^(---\r?\n[\s\S]*?)path:\s*[^\r\n]*\r?\n([\s\S]*?---)/, + "$1$2", + ) + } + + if (currentPath && diskPath) { + return content.replace( + /^(---\r?\n[\s\S]*?)path:\s*[^\r\n]*/, + `$1path: ${diskPath}`, + ) + } + + if (!currentPath && diskPath) { + return content.replace(/^(---\r?\n)/, `$1path: ${diskPath}\n`) + } + + return content +} + +function derivePathFromRelativePath( + relativePath: string, + hasAssets: boolean, +): string | null { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length <= 1) return null + + let directoryParts = parts.slice(0, -1) + if (!hasAssets) { + let path = directoryParts.join("/") + return path || null + } + + let parentParts = directoryParts.slice(0, -1) + let path = parentParts.join("/") + return path || null +} + +function buildLocationFromRelativePath( + baseLocation: ReturnType extends Map< + string, + infer V + > + ? V + : never, + relativePath: string, +) { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return baseLocation + + return { + ...baseLocation, + dirPath: parts.slice(0, -1).join("/"), + filename: parts[parts.length - 1], + } +} + +function supportsFileSystemWatch(): boolean { + return typeof window.FileSystemObserver === "function" +} + +function isBackupSupported(): boolean { + return "showDirectoryPicker" in window +} + +function getSpaceBackupStorageKey(spaceId: string): string { + return `${SPACE_BACKUP_KEY_PREFIX}${spaceId}` +} + +function getSpacesWithBackup( + me: ReturnType< + typeof useAccount + >, + _storageVersion: number, +): string[] { + if (!me.$isLoaded || !me.root?.spaces?.$isLoaded) return [] + + let spaceIds: string[] = [] + for (let space of Array.from(me.root.spaces)) { + if (!space?.$isLoaded) continue + let backupPath = getSpaceBackupPath(space.$jazz.id) + if (backupPath) { + spaceIds.push(space.$jazz.id) + } + } + return spaceIds +} + +async function observeDirectoryChanges( + handle: FileSystemDirectoryHandle, + onChange: () => void, +): Promise<(() => void) | null> { + let Observer = window.FileSystemObserver + if (!Observer) return null + + let observer = new Observer(() => { + onChange() + }) + await observer.observe(handle, { recursive: true }) + return () => observer.disconnect() +} + +async function hashBlob(blob: Blob): Promise { + let buffer = await blob.arrayBuffer() + let hashBuffer = await crypto.subtle.digest("SHA-256", buffer) + let hashArray = Array.from(new Uint8Array(hashBuffer)) + return hashArray + .map(b => b.toString(16).padStart(2, "0")) + .join("") + .slice(0, 16) +} + +function areManifestAssetsEqual( + a: { name: string; hash: string }[], + b: { name: string; hash: string }[], +): boolean { + if (a.length !== b.length) return false + + let sortedA = [...a].sort((left, right) => + left.name.localeCompare(right.name), + ) + let sortedB = [...b].sort((left, right) => + left.name.localeCompare(right.name), + ) + + for (let i = 0; i < sortedA.length; i++) { + if (sortedA[i].name !== sortedB[i].name) return false + if (sortedA[i].hash !== sortedB[i].hash) return false + } + + return true +} + +function wasRecentlyImported(relativePath: string): boolean { + let importedAt = recentImportedRelativePaths.get(relativePath) + if (!importedAt) return false + if (Date.now() - importedAt > RECENT_IMPORT_WINDOW_MS) { + recentImportedRelativePaths.delete(relativePath) + return false + } + return true +} + +function markRecentlyImported(relativePath: string): void { + recentImportedRelativePaths.set(relativePath, Date.now()) +} + +function toTimestamp(value: string | null): number | null { + if (!value) return null + let ms = Date.parse(value) + return Number.isNaN(ms) ? null : ms +} + +function getDocLocationKey(doc: BackupDoc): string { + let path = doc.path ?? "" + let hasAssets = doc.assets.length > 0 ? "assets" : "no-assets" + return `${doc.title}|${path}|${hasAssets}` +} + +async function getSpaceBackupHandle( + spaceId: string, +): Promise { + try { + let handle = await idbGet( + `${HANDLE_STORAGE_KEY}-space-${spaceId}`, + ) + if (!handle) return null + let hasPermission = await verifyPermission(handle) + if (!hasPermission) return null + return handle + } catch { + return null + } } From 48b7e7a574224a74d01128a8ce7a69928b25685d Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Sun, 15 Feb 2026 21:08:49 +0100 Subject: [PATCH 11/19] improve backup and sync logic --- src/lib/backup-scenarios.test.ts | 503 ++++++++++++------------------- src/lib/backup-sync.test.ts | 271 +---------------- src/lib/backup-sync.ts | 3 +- src/lib/backup-test-helpers.ts | 390 ++++++++++++++++++++++++ src/lib/backup.tsx | 88 ++++-- 5 files changed, 638 insertions(+), 617 deletions(-) create mode 100644 src/lib/backup-test-helpers.ts diff --git a/src/lib/backup-scenarios.test.ts b/src/lib/backup-scenarios.test.ts index e11c767..9492eb9 100644 --- a/src/lib/backup-scenarios.test.ts +++ b/src/lib/backup-scenarios.test.ts @@ -1,295 +1,19 @@ import { beforeEach, describe, expect, it } from "vitest" -import { co, Group } from "jazz-tools" +import { co, Group, FileStream } from "jazz-tools" import { createJazzTestAccount, setupJazzTestSync } from "jazz-tools/testing" -import { UserAccount, Document } from "@/schema" +import { UserAccount, Document, Asset, VideoAsset } from "@/schema" import { getPath } from "@/editor/frontmatter" import { getDocumentTitle } from "@/lib/document-utils" import type { BackupDoc } from "./backup-sync" import { readManifest } from "./backup-sync" -import { syncBackup, syncFromBackup } from "./backup" - -class MockWritableFileStream implements FileSystemWritableFileStream { - private stream = new WritableStream() - private saveContent: (content: string) => void - - constructor(saveContent: (content: string) => void) { - this.saveContent = saveContent - } - - get locked(): boolean { - return this.stream.locked - } - - abort(reason?: unknown): Promise { - return this.stream.abort(reason) - } - - close(): Promise { - return Promise.resolve() - } - - getWriter(): WritableStreamDefaultWriter { - return this.stream.getWriter() - } - - seek(_position: number): Promise { - return Promise.resolve() - } - - truncate(_size: number): Promise { - return Promise.resolve() - } - - write(data: string | Blob | ArrayBuffer): Promise - write(data: ArrayBufferView): Promise - write(data: { - type: "write" - data: string | Blob | ArrayBuffer | ArrayBufferView | null - }): Promise - write(data: { type: "seek"; position: number }): Promise - write(data: { type: "truncate"; size: number }): Promise - async write(data: unknown): Promise { - if (typeof data === "string") { - this.saveContent(data) - return - } - - if (data instanceof Blob) { - this.saveContent(await data.text()) - return - } - - if (data instanceof ArrayBuffer) { - this.saveContent(new TextDecoder().decode(new Uint8Array(data))) - return - } - - if (ArrayBuffer.isView(data)) { - let bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength) - this.saveContent(new TextDecoder().decode(bytes)) - return - } - - if (typeof data === "object" && data !== null && "type" in data) { - if ( - data.type === "write" && - "data" in data && - data.data !== undefined && - data.data !== null - ) { - let nestedData = data.data - if (typeof nestedData === "string") { - this.saveContent(nestedData) - return - } - if (nestedData instanceof Blob) { - this.saveContent(await nestedData.text()) - return - } - if (nestedData instanceof ArrayBuffer) { - this.saveContent(new TextDecoder().decode(new Uint8Array(nestedData))) - return - } - if (ArrayBuffer.isView(nestedData)) { - let bytes = new Uint8Array( - nestedData.buffer, - nestedData.byteOffset, - nestedData.byteLength, - ) - this.saveContent(new TextDecoder().decode(bytes)) - } - } - } - } - - get [Symbol.toStringTag](): string { - return "FileSystemWritableFileStream" - } -} - -class MockFileHandle implements FileSystemFileHandle { - kind = "file" as const - name: string - private getContent: () => string - private setContent: (content: string) => void - private getLastModified: () => number - private setLastModified: (lastModified: number) => void - - constructor( - name: string, - getContent: () => string, - setContent: (content: string) => void, - getLastModified: () => number, - setLastModified: (lastModified: number) => void, - ) { - this.name = name - this.getContent = getContent - this.setContent = setContent - this.getLastModified = getLastModified - this.setLastModified = setLastModified - } - - async getFile(): Promise { - return new File([this.getContent()], this.name, { - lastModified: this.getLastModified(), - }) - } - - async createWritable(): Promise { - return new MockWritableFileStream(content => { - this.setContent(content) - this.setLastModified(Date.now()) - }) - } - - isSameEntry(other: FileSystemHandle): Promise { - return Promise.resolve(other.kind === "file" && other.name === this.name) - } - - get [Symbol.toStringTag](): string { - return "FileSystemFileHandle" - } -} - -interface StoredFile { - content: string - lastModified: number -} - -function isDirectoryHandle( - handle: FileSystemHandle | undefined, -): handle is MockDirectoryHandle { - return handle?.kind === "directory" -} - -function isFileHandle( - handle: FileSystemHandle | undefined, -): handle is MockFileHandle { - return handle?.kind === "file" -} - -class MockDirectoryHandle implements FileSystemDirectoryHandle { - kind = "directory" as const - name: string - private children = new Map() - private files = new Map() - - constructor(name: string) { - this.name = name - } - - addFile(name: string, content: string, lastModified = Date.now()) { - this.files.set(name, { content, lastModified }) - let fileHandle = new MockFileHandle( - name, - () => this.files.get(name)?.content ?? "", - updatedContent => { - let entry = this.files.get(name) - this.files.set(name, { - content: updatedContent, - lastModified: entry?.lastModified ?? Date.now(), - }) - }, - () => this.files.get(name)?.lastModified ?? Date.now(), - lastModifiedValue => { - let entry = this.files.get(name) - this.files.set(name, { - content: entry?.content ?? "", - lastModified: lastModifiedValue, - }) - }, - ) - this.children.set(name, fileHandle) - } - - addDirectory(name: string, directory: MockDirectoryHandle) { - this.children.set(name, directory) - } - - entries(): AsyncIterableIterator<[string, FileSystemHandle]> { - let iter = this.children.entries() - return { - async next() { - let result = iter.next() - if (result.done) return { done: true, value: undefined } - return { done: false, value: result.value } - }, - [Symbol.asyncIterator]() { - return this - }, - } - } - - async getFileHandle( - name: string, - options?: { create?: boolean }, - ): Promise { - let handle = this.children.get(name) - if (!isFileHandle(handle)) { - if (options?.create) { - this.addFile(name, "") - let created = this.children.get(name) - if (!isFileHandle(created)) throw new Error("Failed creating file") - return created - } - throw new Error(`File not found: ${name}`) - } - return handle - } - - async getDirectoryHandle( - name: string, - options?: { create?: boolean }, - ): Promise { - let handle = this.children.get(name) - if (!isDirectoryHandle(handle)) { - if (options?.create) { - let created = new MockDirectoryHandle(name) - this.children.set(name, created) - return created - } - throw new Error(`Directory not found: ${name}`) - } - return handle - } - - async removeEntry( - name: string, - options?: { recursive?: boolean }, - ): Promise { - let handle = this.children.get(name) - if (!handle) return - - if (handle.kind === "directory" && !options?.recursive) { - throw new Error("Directory removal requires recursive flag") - } - - this.children.delete(name) - this.files.delete(name) - } - - resolve(): Promise { - return Promise.resolve([this.name]) - } - - queryPermission(): Promise<"granted"> { - return Promise.resolve("granted") - } - - requestPermission(): Promise<"granted"> { - return Promise.resolve("granted") - } - - isSameEntry(other: FileSystemHandle): Promise { - return Promise.resolve( - other.kind === "directory" && other.name === this.name, - ) - } - - get [Symbol.toStringTag](): string { - return "FileSystemDirectoryHandle" - } -} +import { hashContent, syncBackup, syncFromBackup } from "./backup" +import { + MockDirectoryHandle, + basename, + readFileAtPath as readFile, + removeFileAtPath as removeFile, + writeFileAtPath, +} from "./backup-test-helpers" type LoadedAccount = co.loaded type LoadedDoc = co.loaded @@ -347,6 +71,40 @@ describe("backup scenarios", () => { expect(imported).toBeDefined() }) + it("imports asset references with created asset ids", async () => { + await writeFileAtPath( + root, + "Video Doc/Video Doc.md", + "# Video Doc\n\n![Clip](assets/clip.mp4)", + ) + await writeFileAtPath(root, "Video Doc/assets/clip.mp4", "video-bytes") + + let result = await syncFromBackup(root, docs, true) + expect(result.created).toBe(1) + + let imported = getLoadedDocs(docs).find( + d => getDocumentTitle(d) === "Video Doc", + ) + expect(imported).toBeDefined() + if (!imported?.content?.$isLoaded) throw new Error("Doc content not loaded") + + let content = imported.content.toString() + let refMatch = content.match(/!\[[^\]]*\]\(asset:([^)]+)\)/) + expect(refMatch).toBeTruthy() + + let refs = (content.match(/asset:[^)]+/g) ?? []).map(ref => + ref.replace("asset:", ""), + ) + let assetIds = (imported.assets?.$isLoaded ? [...imported.assets] : []) + .filter(asset => asset?.$isLoaded) + .map(asset => asset.$jazz.id) + + expect(assetIds.length).toBeGreaterThan(0) + for (let refId of refs) { + expect(assetIds).toContain(refId) + } + }) + it("renamed in alkalye", async () => { let doc = await createDoc(docs, "# Hello World") await pushToBackup(root, docs) @@ -396,6 +154,109 @@ describe("backup scenarios", () => { expect(getPath(loaded?.content?.toString() ?? "")).toBe("archive") }) + it("matches moved doc by filename when content hashes collide", async () => { + let first = await createDoc(docs, "# Same") + let second = await createDoc(docs, "# Same") + await pushToBackup(root, docs) + + let manifest = await readManifest(root) + if (!manifest) throw new Error("Manifest not found") + + let firstEntry = manifest.entries.find( + entry => entry.docId === first.$jazz.id, + ) + let secondEntry = manifest.entries.find( + entry => entry.docId === second.$jazz.id, + ) + if (!firstEntry || !secondEntry) throw new Error("Manifest entries missing") + + let secondContent = await readFile(root, secondEntry.relativePath) + await removeFile(root, firstEntry.relativePath) + await removeFile(root, secondEntry.relativePath) + + let movedFilename = basename(secondEntry.relativePath) + await writeFileAtPath(root, `archive/${movedFilename}`, secondContent) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBe(1) + expect(result.deleted).toBe(1) + + let firstLoaded = getLoadedDocs(docs).find( + d => d.$jazz.id === first.$jazz.id, + ) + let secondLoaded = getLoadedDocs(docs).find( + d => d.$jazz.id === second.$jazz.id, + ) + + expect(firstLoaded?.deletedAt).toBeTruthy() + expect(secondLoaded?.deletedAt).toBeFalsy() + }) + + it("keeps asset refs stable when pulling updates for docs with assets", async () => { + let { doc, assetId } = await createDocWithVideoAsset(docs, "Asset Sync") + let localContent = + "# Asset Sync\n\n![Clip](assets/clip.mp4)\n\nExternal edit" + let localPath = "Asset Sync.md" + await writeFileAtPath(root, localPath, localContent) + let contentHash = await hashContent( + "# Asset Sync\n\n![Clip](assets/clip.mp4)", + ) + let manifestContent = JSON.stringify( + { + version: 1, + entries: [ + { + docId: doc.$jazz.id, + relativePath: localPath, + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: [{ id: assetId, name: "clip.mp4", hash: "asset-hash" }], + }, + ], + lastSyncAt: new Date().toISOString(), + }, + null, + 2, + ) + await writeFileAtPath(root, ".alkalye-manifest.json", manifestContent) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + if (!loaded?.content?.$isLoaded) throw new Error("Doc content not loaded") + + let content = loaded.content.toString() + expect(content).toContain(`asset:${assetId}`) + expect(content).not.toContain("(assets/") + }) + + it("writes asset ids to manifest entries on backup", async () => { + let backupDocs: BackupDoc[] = [ + { + id: "doc-asset-manifest", + title: "Asset Manifest", + content: "# Asset Manifest\n\n![Clip](asset:asset-1)", + path: null, + updatedAtMs: Date.now(), + assets: [ + { + id: "asset-1", + name: "clip", + blob: new Blob(["video"], { type: "video/mp4" }), + }, + ], + }, + ] + + await syncBackup(root, backupDocs) + + let manifest = await readManifest(root) + expect(manifest).toBeDefined() + expect(manifest?.entries[0].assets[0].id).toBe("asset-1") + }) + it("changed path in alkalye", async () => { let doc = await createDoc(docs, "# Path Doc") await pushToBackup(root, docs) @@ -504,6 +365,44 @@ async function createDoc( return doc } +async function createDocWithVideoAsset( + docs: co.loaded>>, + title: string, +): Promise<{ doc: LoadedDoc; assetId: string }> { + let group = Group.create() + let now = new Date() + let stream = await FileStream.createFromBlob( + new Blob(["video"], { type: "video/mp4" }), + { + owner: group, + }, + ) + let videoAsset = VideoAsset.create( + { + type: "video", + name: "clip", + video: stream, + mimeType: "video/mp4", + createdAt: now, + }, + group, + ) + let content = `# ${title}\n\n![Clip](asset:${videoAsset.$jazz.id})` + let doc = Document.create( + { + version: 1, + content: co.plainText().create(content, group), + assets: co.list(Asset).create([videoAsset], group), + createdAt: now, + updatedAt: now, + }, + group, + ) + docs.$jazz.push(doc) + if (!doc.$isLoaded) throw new Error("Doc failed to load") + return { doc, assetId: videoAsset.$jazz.id } +} + async function pushToBackup( handle: FileSystemDirectoryHandle, docs: co.loaded>>, @@ -537,35 +436,3 @@ async function hasFile( return false } } - -async function readFile( - root: MockDirectoryHandle, - relativePath: string, -): Promise { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length === 0) throw new Error("Empty path") - - let dir: FileSystemDirectoryHandle = root - for (let i = 0; i < parts.length - 1; i++) { - dir = await dir.getDirectoryHandle(parts[i]) - } - - let fileHandle = await dir.getFileHandle(parts[parts.length - 1]) - let file = await fileHandle.getFile() - return file.text() -} - -async function removeFile( - root: MockDirectoryHandle, - relativePath: string, -): Promise { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length === 0) throw new Error("Empty path") - - let dir: FileSystemDirectoryHandle = root - for (let i = 0; i < parts.length - 1; i++) { - dir = await dir.getDirectoryHandle(parts[i]) - } - - await dir.removeEntry(parts[parts.length - 1]) -} diff --git a/src/lib/backup-sync.test.ts b/src/lib/backup-sync.test.ts index 839ed78..3cb81e5 100644 --- a/src/lib/backup-sync.test.ts +++ b/src/lib/backup-sync.test.ts @@ -10,272 +10,11 @@ import { type BackupManifest, type BackupDoc, } from "./backup-sync" - -// ============================================================================= -// Test Helpers -// ============================================================================= - -function createMockFile(content: string, lastModified = Date.now()): File { - return new File([content], "test.md", { lastModified }) -} - -function createMockBlob(content: string, type = "image/png"): Blob { - return new Blob([content], { type }) -} - -class MockWritableFileStream implements FileSystemWritableFileStream { - private stream = new WritableStream() - private saveContent: (content: string) => void - - constructor(saveContent: (content: string) => void) { - this.saveContent = saveContent - } - - get locked(): boolean { - return this.stream.locked - } - - abort(reason?: unknown): Promise { - return this.stream.abort(reason) - } - - close(): Promise { - return Promise.resolve() - } - - getWriter(): WritableStreamDefaultWriter { - return this.stream.getWriter() - } - - seek(_position: number): Promise { - return Promise.resolve() - } - - truncate(_size: number): Promise { - return Promise.resolve() - } - - write(data: string | Blob | ArrayBuffer): Promise - write(data: ArrayBufferView): Promise - write(data: { - type: "write" - data: string | Blob | ArrayBuffer | ArrayBufferView | null - }): Promise - write(data: { type: "seek"; position: number }): Promise - write(data: { type: "truncate"; size: number }): Promise - async write(data: unknown): Promise { - if (typeof data === "string") { - this.saveContent(data) - return - } - - if (data instanceof Blob) { - this.saveContent(await data.text()) - return - } - - if (data instanceof ArrayBuffer) { - let bytes = new Uint8Array(data) - this.saveContent(new TextDecoder().decode(bytes)) - return - } - - if (ArrayBuffer.isView(data)) { - let bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength) - this.saveContent(new TextDecoder().decode(bytes)) - return - } - - if (typeof data === "object" && data !== null && "type" in data) { - if ( - data.type === "write" && - "data" in data && - data.data !== undefined && - data.data !== null - ) { - let nestedData = data.data - if (typeof nestedData === "string") { - this.saveContent(nestedData) - return - } - if (nestedData instanceof Blob) { - this.saveContent(await nestedData.text()) - return - } - if (nestedData instanceof ArrayBuffer) { - let bytes = new Uint8Array(nestedData) - this.saveContent(new TextDecoder().decode(bytes)) - return - } - if (ArrayBuffer.isView(nestedData)) { - let bytes = new Uint8Array( - nestedData.buffer, - nestedData.byteOffset, - nestedData.byteLength, - ) - this.saveContent(new TextDecoder().decode(bytes)) - } - } - } - } - - get [Symbol.toStringTag](): string { - return "FileSystemWritableFileStream" - } -} - -class MockFileHandle implements FileSystemFileHandle { - kind = "file" as const - name: string - private getContent: () => string | undefined - private setContent: (content: string) => void - private initialFile: File - - constructor( - name: string, - getContent: () => string | undefined, - setContent: (content: string) => void, - initialFile: File, - ) { - this.name = name - this.getContent = getContent - this.setContent = setContent - this.initialFile = initialFile - } - - async getFile(): Promise { - let content = this.getContent() - if (content !== undefined) return new File([content], this.name) - return this.initialFile - } - - async createWritable(): Promise { - return new MockWritableFileStream(this.setContent) - } - - isSameEntry(other: FileSystemHandle): Promise { - return Promise.resolve(other.kind === "file" && other.name === this.name) - } - - get [Symbol.toStringTag](): string { - return "FileSystemFileHandle" - } -} - -function isFileHandle( - handle: FileSystemHandle | undefined, -): handle is FileSystemFileHandle { - return handle?.kind === "file" -} - -function isDirectoryHandle( - handle: FileSystemHandle | undefined, -): handle is FileSystemDirectoryHandle { - return handle?.kind === "directory" -} - -// Mock FileSystemDirectoryHandle for testing -class MockDirectoryHandle implements FileSystemDirectoryHandle { - kind = "directory" as const - name: string - private children = new Map() - - constructor(name: string) { - this.name = name - } - - private fileContents = new Map() - - addFile(name: string, file: File) { - let mockHandle = new MockFileHandle( - name, - () => this.fileContents.get(name), - content => this.fileContents.set(name, content), - file, - ) - this.children.set(name, mockHandle) - } - - addDirectory(name: string, dir: FileSystemDirectoryHandle) { - this.children.set(name, dir) - } - - entries(): AsyncIterableIterator<[string, FileSystemHandle]> { - let iter = this.children.entries() - return { - async next() { - let result = iter.next() - return result.done - ? { done: true, value: undefined } - : { done: false, value: result.value } - }, - [Symbol.asyncIterator]() { - return this - }, - } - } - - async getFileHandle( - name: string, - options?: { create?: boolean }, - ): Promise { - let handle = this.children.get(name) - if (!isFileHandle(handle)) { - if (options?.create) { - let mockHandle = new MockFileHandle( - name, - () => this.fileContents.get(name), - content => this.fileContents.set(name, content), - new File([""], name), - ) - this.children.set(name, mockHandle) - return mockHandle - } - throw new Error(`File not found: ${name}`) - } - return handle - } - - async getDirectoryHandle( - name: string, - options?: { create?: boolean }, - ): Promise { - let handle = this.children.get(name) - if (!isDirectoryHandle(handle)) { - if (options?.create) { - let newDir = new MockDirectoryHandle(name) - this.children.set(name, newDir) - return newDir - } - throw new Error(`Directory not found: ${name}`) - } - return handle - } - - removeEntry(): Promise { - return Promise.resolve() - } - - resolve(): Promise { - return Promise.resolve([this.name]) - } - - isSameEntry(other: FileSystemHandle): Promise { - return Promise.resolve(this.name === other.name) - } - - queryPermission(): Promise<"granted"> { - return Promise.resolve("granted") - } - - requestPermission(): Promise<"granted"> { - return Promise.resolve("granted") - } - - get [Symbol.toStringTag](): string { - return "FileSystemDirectoryHandle" - } -} +import { + MockDirectoryHandle, + createMockBlob, + createMockFile, +} from "./backup-test-helpers" // ============================================================================= // Compute Doc Locations diff --git a/src/lib/backup-sync.ts b/src/lib/backup-sync.ts index ad17394..63c65bf 100644 --- a/src/lib/backup-sync.ts +++ b/src/lib/backup-sync.ts @@ -44,7 +44,7 @@ interface ManifestEntry { locationKey?: string contentHash: string lastSyncedAt: string - assets: { name: string; hash: string }[] + assets: { id?: string; name: string; hash: string }[] } interface BackupManifest { @@ -62,6 +62,7 @@ interface ScannedFile { } let manifestAssetSchema = z.object({ + id: z.string().optional(), name: z.string(), hash: z.string(), }) diff --git a/src/lib/backup-test-helpers.ts b/src/lib/backup-test-helpers.ts new file mode 100644 index 0000000..d2e5572 --- /dev/null +++ b/src/lib/backup-test-helpers.ts @@ -0,0 +1,390 @@ +export { + MockDirectoryHandle, + createMockBlob, + createMockFile, + readFileAtPath, + removeFileAtPath, + writeFileAtPath, + basename, +} + +interface StoredFile { + content: string | null + source: File | null + lastModified: number + type: string +} + +function createMockFile(content: string, lastModified = Date.now()): File { + return new File([content], "test.md", { lastModified }) +} + +function createMockBlob(content: string, type = "image/png"): Blob { + return new Blob([content], { type }) +} + +class MockWritableFileStream implements FileSystemWritableFileStream { + private stream = new WritableStream() + private saveContent: (content: string) => void + + constructor(saveContent: (content: string) => void) { + this.saveContent = saveContent + } + + get locked(): boolean { + return this.stream.locked + } + + abort(reason?: unknown): Promise { + return this.stream.abort(reason) + } + + close(): Promise { + return Promise.resolve() + } + + getWriter(): WritableStreamDefaultWriter { + return this.stream.getWriter() + } + + seek(_position: number): Promise { + return Promise.resolve() + } + + truncate(_size: number): Promise { + return Promise.resolve() + } + + write(data: string | Blob | ArrayBuffer): Promise + write(data: ArrayBufferView): Promise + write(data: { + type: "write" + data: string | Blob | ArrayBuffer | ArrayBufferView | null + }): Promise + write(data: { type: "seek"; position: number }): Promise + write(data: { type: "truncate"; size: number }): Promise + async write(data: unknown): Promise { + if (typeof data === "string") { + this.saveContent(data) + return + } + + if (data instanceof Blob) { + this.saveContent(await data.text()) + return + } + + if (data instanceof ArrayBuffer) { + let bytes = new Uint8Array(data) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + + if (ArrayBuffer.isView(data)) { + let bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + + if (typeof data === "object" && data !== null && "type" in data) { + if ( + data.type === "write" && + "data" in data && + data.data !== undefined && + data.data !== null + ) { + let nestedData = data.data + if (typeof nestedData === "string") { + this.saveContent(nestedData) + return + } + if (nestedData instanceof Blob) { + this.saveContent(await nestedData.text()) + return + } + if (nestedData instanceof ArrayBuffer) { + let bytes = new Uint8Array(nestedData) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + if (ArrayBuffer.isView(nestedData)) { + let bytes = new Uint8Array( + nestedData.buffer, + nestedData.byteOffset, + nestedData.byteLength, + ) + this.saveContent(new TextDecoder().decode(bytes)) + } + } + } + } + + get [Symbol.toStringTag](): string { + return "FileSystemWritableFileStream" + } +} + +class MockFileHandle implements FileSystemFileHandle { + kind = "file" as const + name: string + private getStoredFile: () => StoredFile | undefined + private setStoredFile: (file: StoredFile) => void + + constructor( + name: string, + getStoredFile: () => StoredFile | undefined, + setStoredFile: (file: StoredFile) => void, + ) { + this.name = name + this.getStoredFile = getStoredFile + this.setStoredFile = setStoredFile + } + + async getFile(): Promise { + let file = this.getStoredFile() + if (!file) { + return new File([""], this.name) + } + if (file.content === null && file.source) { + return file.source + } + return new File([file.content ?? ""], this.name, { + lastModified: file.lastModified, + type: file.type, + }) + } + + async createWritable(): Promise { + return new MockWritableFileStream(content => { + let current = this.getStoredFile() + this.setStoredFile({ + content, + source: null, + lastModified: Date.now(), + type: current?.type ?? inferMimeType(this.name), + }) + }) + } + + isSameEntry(other: FileSystemHandle): Promise { + return Promise.resolve(other.kind === "file" && other.name === this.name) + } + + get [Symbol.toStringTag](): string { + return "FileSystemFileHandle" + } +} + +function isFileHandle( + handle: FileSystemHandle | undefined, +): handle is FileSystemFileHandle { + return handle?.kind === "file" +} + +function isDirectoryHandle( + handle: FileSystemHandle | undefined, +): handle is FileSystemDirectoryHandle { + return handle?.kind === "directory" +} + +class MockDirectoryHandle implements FileSystemDirectoryHandle { + kind = "directory" as const + name: string + private children = new Map() + private files = new Map() + + constructor(name: string) { + this.name = name + } + + addFile( + name: string, + fileOrContent: File | string, + lastModified = Date.now(), + ) { + let file = + typeof fileOrContent === "string" + ? { + content: fileOrContent, + source: null, + lastModified, + type: inferMimeType(name), + } + : { + content: null, + source: fileOrContent, + lastModified: fileOrContent.lastModified || lastModified, + type: fileOrContent.type || inferMimeType(name), + } + + this.files.set(name, file) + this.children.set( + name, + new MockFileHandle( + name, + () => this.files.get(name), + updated => this.files.set(name, updated), + ), + ) + } + + addDirectory(name: string, directory: MockDirectoryHandle) { + this.children.set(name, directory) + } + + entries(): AsyncIterableIterator<[string, FileSystemHandle]> { + let iter = this.children.entries() + return { + async next() { + let result = iter.next() + if (result.done) return { done: true, value: undefined } + return { done: false, value: result.value } + }, + [Symbol.asyncIterator]() { + return this + }, + } + } + + async getFileHandle( + name: string, + options?: { create?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!isFileHandle(handle)) { + if (options?.create) { + this.addFile(name, "") + let created = this.children.get(name) + if (!isFileHandle(created)) throw new Error("Failed creating file") + return created + } + throw new Error(`File not found: ${name}`) + } + return handle + } + + async getDirectoryHandle( + name: string, + options?: { create?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!isDirectoryHandle(handle)) { + if (options?.create) { + let created = new MockDirectoryHandle(name) + this.children.set(name, created) + return created + } + throw new Error(`Directory not found: ${name}`) + } + return handle + } + + async removeEntry( + name: string, + options?: { recursive?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!handle) return + + if (handle.kind === "directory" && !options?.recursive) { + throw new Error("Directory removal requires recursive flag") + } + + this.children.delete(name) + this.files.delete(name) + } + + resolve(): Promise { + return Promise.resolve([this.name]) + } + + queryPermission(): Promise<"granted"> { + return Promise.resolve("granted") + } + + requestPermission(): Promise<"granted"> { + return Promise.resolve("granted") + } + + isSameEntry(other: FileSystemHandle): Promise { + return Promise.resolve( + other.kind === "directory" && other.name === this.name, + ) + } + + get [Symbol.toStringTag](): string { + return "FileSystemDirectoryHandle" + } +} + +async function readFileAtPath( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + + let fileHandle = await dir.getFileHandle(parts[parts.length - 1]) + let file = await fileHandle.getFile() + return file.text() +} + +async function removeFileAtPath( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + + await dir.removeEntry(parts[parts.length - 1]) +} + +async function writeFileAtPath( + root: MockDirectoryHandle, + relativePath: string, + content: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i], { create: true }) + } + + let fileHandle = await dir.getFileHandle(parts[parts.length - 1], { + create: true, + }) + let writable = await fileHandle.createWritable() + await writable.write(content) + await writable.close() +} + +function basename(relativePath: string): string { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return relativePath + return parts[parts.length - 1] +} + +function inferMimeType(filename: string): string { + let lowered = filename.toLowerCase() + if (lowered.endsWith(".mp4")) return "video/mp4" + if (lowered.endsWith(".webm")) return "video/webm" + if (lowered.endsWith(".png")) return "image/png" + if (lowered.endsWith(".jpg") || lowered.endsWith(".jpeg")) return "image/jpeg" + if (lowered.endsWith(".gif")) return "image/gif" + if (lowered.endsWith(".webp")) return "image/webp" + if (lowered.endsWith(".svg")) return "image/svg+xml" + if (lowered.endsWith(".md")) return "text/markdown" + return "application/octet-stream" +} diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx index 4ced114..9f9b9dc 100644 --- a/src/lib/backup.tsx +++ b/src/lib/backup.tsx @@ -26,6 +26,7 @@ import { writeManifest, transformContentForImport, type BackupDoc, + type ManifestEntry, type ScannedFile, } from "@/lib/backup-sync" @@ -757,6 +758,7 @@ async function syncFromBackup( manifest, scannedByPath, matchedManifestDocIds, + file, contentHash, ) if (movedEntry) { @@ -789,6 +791,7 @@ async function syncFromBackup( let didUpdate = await updateDocFromFile( file, manifestEntry.docId, + manifestEntry, targetDocs, ) if (didUpdate) { @@ -1127,7 +1130,7 @@ async function performSyncBackup( locationKey: string contentHash: string lastSyncedAt: string - assets: { name: string; hash: string }[] + assets: { id: string; name: string; hash: string }[] }[] = [] let hasFilesystemChanges = false let nowIso = new Date().toISOString() @@ -1161,10 +1164,11 @@ async function performSyncBackup( let exportedContent = transformContentForBackup(doc.content, loc.assetFiles) let contentHash = await hashContent(exportedContent) let relativePath = finalRelativePath - let assets: { name: string; hash: string }[] = [] + let assets: { id: string; name: string; hash: string }[] = [] for (let asset of doc.assets) { let filename = loc.assetFiles.get(asset.id)! assets.push({ + id: asset.id, name: filename, hash: await hashBlob(asset.blob), }) @@ -1295,19 +1299,6 @@ async function createDocFromFile( targetDocs: DocumentList, listOwner: Group | Account, ): Promise { - // Transform asset references back to asset: format - let assetFiles = new Map() - for (let asset of file.assets) { - let id = crypto.randomUUID() - assetFiles.set(id, asset.name) - } - let transformedContent = transformContentForImport(file.content, assetFiles) - let content = applyPathFromRelativePath( - transformedContent, - file.relativePath, - file.assets.length > 0, - ) - // Create doc-specific group with list owner as parent let docGroup = Group.create() if (listOwner instanceof Group) { @@ -1318,12 +1309,9 @@ async function createDocFromFile( // Create assets let docAssets: co.loaded[] = [] + let assetFilesById = new Map() for (let assetFile of file.assets) { let isVideo = assetFile.blob.type.startsWith("video/") - let id = [...assetFiles.entries()].find( - ([, name]) => name === assetFile.name, - )?.[0] - if (!id) continue if (isVideo) { let video = await FileStream.createFromBlob(assetFile.blob, { @@ -1340,6 +1328,7 @@ async function createDocFromFile( docGroup, ) docAssets.push(asset) + assetFilesById.set(asset.$jazz.id, assetFile.name) } else { let image = await createImage(assetFile.blob, { owner: docGroup, @@ -1355,9 +1344,20 @@ async function createDocFromFile( docGroup, ) docAssets.push(asset) + assetFilesById.set(asset.$jazz.id, assetFile.name) } } + let transformedContent = transformContentForImport( + file.content, + assetFilesById, + ) + let content = applyPathFromRelativePath( + transformedContent, + file.relativePath, + file.assets.length > 0, + ) + let newDoc = Document.create( { version: 1, @@ -1379,6 +1379,7 @@ async function createDocFromFile( async function updateDocFromFile( file: ScannedFile, docId: string, + manifestEntry: ManifestEntry, targetDocs: DocumentList, ): Promise { let doc = targetDocs.find( @@ -1389,14 +1390,9 @@ async function updateDocFromFile( } // Update content - let assetFiles = new Map() - for (let asset of file.assets) { - let id = crypto.randomUUID() - assetFiles.set(id, asset.name) - } - let transformedContent = transformContentForImport(file.content, assetFiles) + let assetFilesById = getAssetFilesByIdFromManifest(manifestEntry) let content = applyPathFromRelativePath( - transformedContent, + transformContentForImport(file.content, assetFilesById), file.relativePath, file.assets.length > 0, ) @@ -1424,19 +1420,46 @@ function findMovedManifestEntry( manifest: Awaited>, scannedByPath: Map, matchedManifestDocIds: Set, + file: ScannedFile, contentHash: string, ) { if (!manifest) return null + let candidates = manifest.entries.filter(entry => { + if (matchedManifestDocIds.has(entry.docId)) return false + if (scannedByPath.has(entry.relativePath)) return false + if (entry.contentHash !== contentHash) return false + return true + }) + if (candidates.length === 0) return null + if (candidates.length === 1) return candidates[0] - for (let entry of manifest.entries) { - if (matchedManifestDocIds.has(entry.docId)) continue - if (scannedByPath.has(entry.relativePath)) continue - if (entry.contentHash === contentHash) return entry + let matchingBasename = candidates.filter(entry => { + return getFilename(entry.relativePath) === getFilename(file.relativePath) + }) + if (matchingBasename.length === 1) { + return matchingBasename[0] } return null } +function getFilename(relativePath: string): string { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return relativePath + return parts[parts.length - 1] +} + +function getAssetFilesByIdFromManifest( + manifestEntry: ManifestEntry, +): Map { + let filesById = new Map() + for (let asset of manifestEntry.assets) { + if (!asset.id) continue + filesById.set(asset.id, asset.name) + } + return filesById +} + function applyPathFromRelativePath( content: string, relativePath: string, @@ -1567,8 +1590,8 @@ async function hashBlob(blob: Blob): Promise { } function areManifestAssetsEqual( - a: { name: string; hash: string }[], - b: { name: string; hash: string }[], + a: { id?: string; name: string; hash: string }[], + b: { id?: string; name: string; hash: string }[], ): boolean { if (a.length !== b.length) return false @@ -1580,6 +1603,7 @@ function areManifestAssetsEqual( ) for (let i = 0; i < sortedA.length; i++) { + if ((sortedA[i].id ?? null) !== (sortedB[i].id ?? null)) return false if (sortedA[i].name !== sortedB[i].name) return false if (sortedA[i].hash !== sortedB[i].hash) return false } From f692a50eed3f4b64066ab307f98c48de91abe4ab Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Sun, 15 Feb 2026 21:26:30 +0100 Subject: [PATCH 12/19] implement skipped tests --- src/lib/backup-scenarios.test.ts | 113 +++++++++++ src/lib/backup-test-helpers.ts | 118 +++++------ src/lib/backup.tsx | 333 +++++++++++++++++++++++++++---- 3 files changed, 462 insertions(+), 102 deletions(-) diff --git a/src/lib/backup-scenarios.test.ts b/src/lib/backup-scenarios.test.ts index 9492eb9..38a0b8b 100644 --- a/src/lib/backup-scenarios.test.ts +++ b/src/lib/backup-scenarios.test.ts @@ -232,6 +232,71 @@ describe("backup scenarios", () => { expect(content).not.toContain("(assets/") }) + it("imports asset binary updates when markdown is unchanged", async () => { + let { doc, assetId } = await createDocWithVideoAsset(docs, "Binary Update") + let localPath = "Binary Update/Binary Update.md" + let localContent = "# Binary Update\n\n![Clip](assets/clip.mp4)" + await writeFileAtPath(root, localPath, localContent) + await writeFileAtPath( + root, + "Binary Update/assets/clip.mp4", + "new-video-bytes", + ) + + let contentHash = await hashContent(localContent) + let previousAssetHash = await hashContent("old-video-bytes") + await writeFileAtPath( + root, + ".alkalye-manifest.json", + JSON.stringify( + { + version: 1, + entries: [ + { + docId: doc.$jazz.id, + relativePath: localPath, + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: [ + { + id: assetId, + name: "clip.mp4", + hash: previousAssetHash, + }, + ], + }, + ], + lastSyncAt: new Date().toISOString(), + }, + null, + 2, + ), + ) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + let updatedAsset = loaded?.assets?.$isLoaded + ? [...loaded.assets].find( + asset => asset?.$isLoaded && asset.$jazz.id === assetId, + ) + : undefined + expect(updatedAsset?.$isLoaded).toBe(true) + if ( + !updatedAsset?.$isLoaded || + updatedAsset.type !== "video" || + !updatedAsset.video?.$isLoaded + ) { + throw new Error("Updated video asset not loaded") + } + + let blob = await updatedAsset.video.toBlob() + if (!blob) throw new Error("Updated video blob missing") + expect(await blob.text()).toBe("new-video-bytes") + }) + it("writes asset ids to manifest entries on backup", async () => { let backupDocs: BackupDoc[] = [ { @@ -257,6 +322,54 @@ describe("backup scenarios", () => { expect(manifest?.entries[0].assets[0].id).toBe("asset-1") }) + it("removes stale files from assets folders during backup", async () => { + let first: BackupDoc[] = [ + { + id: "doc-assets-prune", + title: "Assets Prune", + content: + "# Assets Prune\n\n![One](asset:asset-1)\n![Two](asset:asset-2)", + path: null, + updatedAtMs: Date.now(), + assets: [ + { + id: "asset-1", + name: "one", + blob: new Blob(["one"], { type: "image/png" }), + }, + { + id: "asset-2", + name: "two", + blob: new Blob(["two"], { type: "image/png" }), + }, + ], + }, + ] + await syncBackup(root, first) + expect(await hasFile(root, "Assets Prune/assets/two.png")).toBe(true) + + let second: BackupDoc[] = [ + { + id: "doc-assets-prune", + title: "Assets Prune", + content: "# Assets Prune\n\n![One](asset:asset-1)", + path: null, + updatedAtMs: Date.now() + 1, + assets: [ + { + id: "asset-1", + name: "one", + blob: new Blob(["one"], { type: "image/png" }), + }, + ], + }, + ] + await syncBackup(root, second) + + expect(await hasFile(root, "Assets Prune/assets/one.png")).toBe(true) + expect(await hasFile(root, "Assets Prune/assets/two.png")).toBe(false) + }) + it("changed path in alkalye", async () => { let doc = await createDoc(docs, "# Path Doc") await pushToBackup(root, docs) diff --git a/src/lib/backup-test-helpers.ts b/src/lib/backup-test-helpers.ts index d2e5572..87f33be 100644 --- a/src/lib/backup-test-helpers.ts +++ b/src/lib/backup-test-helpers.ts @@ -23,6 +23,65 @@ function createMockBlob(content: string, type = "image/png"): Blob { return new Blob([content], { type }) } +async function readFileAtPath( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + + let fileHandle = await dir.getFileHandle(parts[parts.length - 1]) + let file = await fileHandle.getFile() + return file.text() +} + +async function removeFileAtPath( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + + await dir.removeEntry(parts[parts.length - 1]) +} + +async function writeFileAtPath( + root: MockDirectoryHandle, + relativePath: string, + content: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i], { create: true }) + } + + let fileHandle = await dir.getFileHandle(parts[parts.length - 1], { + create: true, + }) + let writable = await fileHandle.createWritable() + await writable.write(content) + await writable.close() +} + +function basename(relativePath: string): string { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return relativePath + return parts[parts.length - 1] +} + class MockWritableFileStream implements FileSystemWritableFileStream { private stream = new WritableStream() private saveContent: (content: string) => void @@ -317,65 +376,6 @@ class MockDirectoryHandle implements FileSystemDirectoryHandle { } } -async function readFileAtPath( - root: MockDirectoryHandle, - relativePath: string, -): Promise { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length === 0) throw new Error("Empty path") - - let dir: FileSystemDirectoryHandle = root - for (let i = 0; i < parts.length - 1; i++) { - dir = await dir.getDirectoryHandle(parts[i]) - } - - let fileHandle = await dir.getFileHandle(parts[parts.length - 1]) - let file = await fileHandle.getFile() - return file.text() -} - -async function removeFileAtPath( - root: MockDirectoryHandle, - relativePath: string, -): Promise { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length === 0) throw new Error("Empty path") - - let dir: FileSystemDirectoryHandle = root - for (let i = 0; i < parts.length - 1; i++) { - dir = await dir.getDirectoryHandle(parts[i]) - } - - await dir.removeEntry(parts[parts.length - 1]) -} - -async function writeFileAtPath( - root: MockDirectoryHandle, - relativePath: string, - content: string, -): Promise { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length === 0) throw new Error("Empty path") - - let dir: FileSystemDirectoryHandle = root - for (let i = 0; i < parts.length - 1; i++) { - dir = await dir.getDirectoryHandle(parts[i], { create: true }) - } - - let fileHandle = await dir.getFileHandle(parts[parts.length - 1], { - create: true, - }) - let writable = await fileHandle.createWritable() - await writable.write(content) - await writable.close() -} - -function basename(relativePath: string): string { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length === 0) return relativePath - return parts[parts.length - 1] -} - function inferMimeType(filename: string): string { let lowered = filename.toLowerCase() if (lowered.endsWith(".mp4")) return "video/mp4" diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx index 9f9b9dc..cdad7c9 100644 --- a/src/lib/backup.tsx +++ b/src/lib/backup.tsx @@ -149,6 +149,11 @@ interface SyncFromBackupResult { errors: string[] } +interface ScannedAssetHash { + name: string + hash: string +} + let backupQuery = { root: { documents: { @@ -745,6 +750,7 @@ async function syncFromBackup( for (let file of scannedFiles) { try { let contentHash = await hashContent(file.content) + let scannedAssetHashes = await hashScannedAssets(file.assets) let manifestEntry = manifestByPath.get(file.relativePath) if (manifestEntry) { matchedManifestDocIds.add(manifestEntry.docId) @@ -780,7 +786,8 @@ async function syncFromBackup( result.created++ } else if ( manifestEntry.contentHash !== contentHash || - manifestEntry.relativePath !== file.relativePath + manifestEntry.relativePath !== file.relativePath || + !areScannedAssetsInSync(manifestEntry.assets, scannedAssetHashes) ) { preferredRelativePathByDocId.set(manifestEntry.docId, file.relativePath) // File changed or moved - update document @@ -1244,6 +1251,39 @@ async function cleanupOrphanedFiles( docs, docLocations, ) + let expectedAssetFilesByDir = new Map>() + for (let doc of docs) { + let location = docLocations.get(doc.id) + if (!location || !location.hasOwnFolder) continue + let assetsPath = location.dirPath ? `${location.dirPath}/assets` : "assets" + expectedAssetFilesByDir.set( + assetsPath, + new Set(location.assetFiles.values()), + ) + } + + async function cleanAssetsDirectory( + dir: FileSystemDirectoryHandle, + path: string, + ): Promise { + let hasContent = false + let expected = expectedAssetFilesByDir.get(path) ?? new Set() + + for await (let [name, child] of dir.entries()) { + if (name.startsWith(".")) continue + if (child.kind === "directory") { + await dir.removeEntry(name, { recursive: true }) + continue + } + if (expected.has(name)) { + hasContent = true + continue + } + await deleteFile(dir, name) + } + + return hasContent + } async function cleanDir( dir: FileSystemDirectoryHandle, @@ -1255,9 +1295,10 @@ async function cleanupOrphanedFiles( for (let subdir of subdirs) { let subPath = path ? `${path}/${subdir}` : subdir - // Skip assets folders that belong to a doc if (subdir === "assets" && expectedPaths.has(subPath)) { - hasContent = true + let subHandle = await dir.getDirectoryHandle(subdir) + let assetsHasContent = await cleanAssetsDirectory(subHandle, subPath) + if (assetsHasContent) hasContent = true continue } @@ -1311,41 +1352,9 @@ async function createDocFromFile( let docAssets: co.loaded[] = [] let assetFilesById = new Map() for (let assetFile of file.assets) { - let isVideo = assetFile.blob.type.startsWith("video/") - - if (isVideo) { - let video = await FileStream.createFromBlob(assetFile.blob, { - owner: docGroup, - }) - let asset = VideoAsset.create( - { - type: "video", - name: assetFile.name.replace(/\.[^.]+$/, ""), - video, - mimeType: "video/mp4", - createdAt: now, - }, - docGroup, - ) - docAssets.push(asset) - assetFilesById.set(asset.$jazz.id, assetFile.name) - } else { - let image = await createImage(assetFile.blob, { - owner: docGroup, - maxSize: 2048, - }) - let asset = ImageAsset.create( - { - type: "image", - name: assetFile.name.replace(/\.[^.]+$/, ""), - image, - createdAt: now, - }, - docGroup, - ) - docAssets.push(asset) - assetFilesById.set(asset.$jazz.id, assetFile.name) - } + let asset = await createAssetFromBlob(assetFile, docGroup, now) + docAssets.push(asset) + assetFilesById.set(asset.$jazz.id, assetFile.name) } let transformedContent = transformContentForImport( @@ -1389,8 +1398,7 @@ async function updateDocFromFile( return false } - // Update content - let assetFilesById = getAssetFilesByIdFromManifest(manifestEntry) + let assetFilesById = await syncDocAssetsFromFile(doc, file, manifestEntry) let content = applyPathFromRelativePath( transformContentForImport(file.content, assetFilesById), file.relativePath, @@ -1399,12 +1407,218 @@ async function updateDocFromFile( doc.content.$jazz.applyDiff(content) doc.$jazz.set("updatedAt", new Date()) - - // TODO: Handle asset updates (add/remove/replace assets) - // For now, we just update the content. Asset changes require more complex logic. return true } +async function syncDocAssetsFromFile( + doc: LoadedDocument, + file: ScannedFile, + manifestEntry: ManifestEntry, +): Promise> { + if (file.assets.length === 0) { + let manifestFilesById = getAssetFilesByIdFromManifest(manifestEntry) + let keepsManifestRefs = Array.from(manifestFilesById.values()).some( + filename => file.content.includes(`assets/${filename}`), + ) + if (keepsManifestRefs) { + return manifestFilesById + } + if (doc.assets?.$isLoaded) { + for (let i = doc.assets.length - 1; i >= 0; i--) { + doc.assets.$jazz.splice(i, 1) + } + } + return new Map() + } + + if (!doc.assets) { + doc.$jazz.set("assets", co.list(Asset).create([], doc.$jazz.owner)) + } + if (!doc.assets?.$isLoaded) { + return getAssetFilesByIdFromManifest(manifestEntry) + } + + let currentAssets = Array.from(doc.assets).filter( + (asset): asset is co.loaded => asset?.$isLoaded === true, + ) + let assetsById = new Map(currentAssets.map(asset => [asset.$jazz.id, asset])) + let fileAssetsWithHash = await Promise.all( + file.assets.map(async asset => ({ + name: asset.name, + blob: asset.blob, + hash: await hashBlob(asset.blob), + })), + ) + + let manifestByName = new Map( + manifestEntry.assets.map(asset => [asset.name, asset]), + ) + let manifestByHash = new Map() + for (let asset of manifestEntry.assets) { + if (!manifestByHash.has(asset.hash)) { + manifestByHash.set(asset.hash, []) + } + manifestByHash.get(asset.hash)?.push(asset) + } + + let keepAssetIds = new Set() + let assetFilesById = new Map() + + for (let fileAsset of fileAssetsWithHash) { + let matchedByName = manifestByName.get(fileAsset.name) + let matchedId = + matchedByName?.id && assetsById.has(matchedByName.id) + ? matchedByName.id + : null + + if (!matchedId) { + let byHash = manifestByHash.get(fileAsset.hash) ?? [] + for (let candidate of byHash) { + if (!candidate.id || keepAssetIds.has(candidate.id)) continue + if (!assetsById.has(candidate.id)) continue + matchedId = candidate.id + break + } + } + + if (matchedId) { + let existing = assetsById.get(matchedId) + if (!existing) continue + + let shouldUpdateBinary = + matchedByName?.id === matchedId + ? matchedByName.hash !== fileAsset.hash + : false + + if (shouldUpdateBinary) { + matchedId = await syncExistingAssetFromFile( + doc, + existing, + matchedId, + fileAsset, + ) + } + + let updatedAsset = doc.assets.find( + asset => asset?.$isLoaded && asset.$jazz.id === matchedId, + ) + if (!updatedAsset) continue + assetsById.set(matchedId, updatedAsset) + if (updatedAsset.name !== removeExtension(fileAsset.name)) { + updatedAsset.$jazz.applyDiff({ name: removeExtension(fileAsset.name) }) + } + + keepAssetIds.add(matchedId) + assetFilesById.set(matchedId, fileAsset.name) + continue + } + + let created = await createAssetFromBlob( + fileAsset, + doc.$jazz.owner, + new Date(), + ) + doc.assets.$jazz.push(created) + keepAssetIds.add(created.$jazz.id) + assetFilesById.set(created.$jazz.id, fileAsset.name) + } + + for (let i = doc.assets.length - 1; i >= 0; i--) { + let asset = doc.assets[i] + if (!asset?.$isLoaded) continue + if (keepAssetIds.has(asset.$jazz.id)) continue + doc.assets.$jazz.splice(i, 1) + } + + if (fileAssetsWithHash.length === 0) { + return new Map() + } + + return assetFilesById +} + +async function syncExistingAssetFromFile( + doc: LoadedDocument, + existing: co.loaded, + assetId: string, + fileAsset: { name: string; blob: Blob }, +): Promise { + let nextType = fileAsset.blob.type.startsWith("video/") ? "video" : "image" + if (existing.type !== nextType) { + let index = + doc.assets?.findIndex(asset => asset?.$jazz.id === assetId) ?? -1 + if (index === -1 || !doc.assets) return assetId + doc.assets.$jazz.splice(index, 1) + let replacement = await createAssetFromBlob( + fileAsset, + doc.$jazz.owner, + existing.createdAt, + ) + doc.assets.$jazz.push(replacement) + return replacement.$jazz.id + } + + if (existing.type === "video") { + let stream = await FileStream.createFromBlob(fileAsset.blob, { + owner: doc.$jazz.owner, + }) + existing.$jazz.applyDiff({ + name: removeExtension(fileAsset.name), + video: stream, + mimeType: fileAsset.blob.type || "video/mp4", + }) + return assetId + } + + let image = await createImage(fileAsset.blob, { + owner: doc.$jazz.owner, + maxSize: 2048, + }) + existing.$jazz.applyDiff({ name: removeExtension(fileAsset.name), image }) + return assetId +} + +async function createAssetFromBlob( + assetFile: { name: string; blob: Blob }, + owner: Group, + now: Date, +) { + let isVideo = assetFile.blob.type.startsWith("video/") + if (isVideo) { + let video = await FileStream.createFromBlob(assetFile.blob, { + owner, + }) + return VideoAsset.create( + { + type: "video", + name: removeExtension(assetFile.name), + video, + mimeType: assetFile.blob.type || "video/mp4", + createdAt: now, + }, + owner, + ) + } + + let image = await createImage(assetFile.blob, { + owner, + maxSize: 2048, + }) + return ImageAsset.create( + { + type: "image", + name: removeExtension(assetFile.name), + image, + createdAt: now, + }, + owner, + ) +} + +function removeExtension(filename: string): string { + return filename.replace(/\.[^.]+$/, "") +} + function isDocumentList(value: unknown): value is DocumentList { if (typeof value !== "object" || value === null) return false return "$jazz" in value && "find" in value @@ -1589,6 +1803,39 @@ async function hashBlob(blob: Blob): Promise { .slice(0, 16) } +async function hashScannedAssets( + assets: { name: string; blob: Blob }[], +): Promise { + let hashed = await Promise.all( + assets.map(async asset => ({ + name: asset.name, + hash: await hashBlob(asset.blob), + })), + ) + return hashed +} + +function areScannedAssetsInSync( + manifestAssets: { id?: string; name: string; hash: string }[], + scannedAssets: ScannedAssetHash[], +): boolean { + if (manifestAssets.length !== scannedAssets.length) return false + + let sortedManifest = [...manifestAssets].sort((a, b) => + a.name.localeCompare(b.name), + ) + let sortedScanned = [...scannedAssets].sort((a, b) => + a.name.localeCompare(b.name), + ) + + for (let i = 0; i < sortedManifest.length; i++) { + if (sortedManifest[i].name !== sortedScanned[i].name) return false + if (sortedManifest[i].hash !== sortedScanned[i].hash) return false + } + + return true +} + function areManifestAssetsEqual( a: { id?: string; name: string; hash: string }[], b: { id?: string; name: string; hash: string }[], From 916a79486c5b9e1b4a31d0f38528b290f0acc06f Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Sun, 15 Feb 2026 22:04:07 +0100 Subject: [PATCH 13/19] split large modules, improve test coverage and robustness --- src/lib/backup-engine.ts | 971 ++++++++++++ src/lib/backup-scenarios.test.ts | 202 ++- src/lib/backup-settings-ui.tsx | 299 ++++ src/lib/backup-storage.ts | 304 ++++ src/lib/backup-subscribers.tsx | 346 +++++ src/lib/backup-sync.test.ts | 158 +- src/lib/backup-sync.ts | 84 +- src/lib/backup-test-helpers.ts | 105 +- src/lib/backup.test.ts | 2 +- src/lib/backup.tsx | 1901 ----------------------- src/main.tsx | 5 +- src/routes/settings.tsx | 2 +- src/routes/spaces.$spaceId.settings.tsx | 2 +- 13 files changed, 2412 insertions(+), 1969 deletions(-) create mode 100644 src/lib/backup-engine.ts create mode 100644 src/lib/backup-settings-ui.tsx create mode 100644 src/lib/backup-storage.ts create mode 100644 src/lib/backup-subscribers.tsx delete mode 100644 src/lib/backup.tsx diff --git a/src/lib/backup-engine.ts b/src/lib/backup-engine.ts new file mode 100644 index 0000000..5ac2c60 --- /dev/null +++ b/src/lib/backup-engine.ts @@ -0,0 +1,971 @@ +import { co, Group, Account, FileStream } from "jazz-tools" +import { createImage } from "jazz-tools/media" +import { Document, Asset, ImageAsset, VideoAsset } from "@/schema" +import { getDocumentTitle } from "@/lib/document-utils" +import { getPath, parseFrontmatter } from "@/editor/frontmatter" +import { + computeDocLocations, + transformContentForBackup, + computeExpectedStructure, + scanBackupFolder, + readManifest, + writeManifest, + transformContentForImport, + type BackupDoc, + type ManifestEntry, + type ScannedFile, +} from "@/lib/backup-sync" + +export { + hashContent, + syncBackup, + syncFromBackup, + prepareBackupDoc, + type LoadedDocument, + type DocumentList, +} + +type LoadedDocument = co.loaded< + typeof Document, + { content: true; assets: { $each: { image: true; video: true } } } +> + +type DocumentList = co.loaded>> + +interface SyncFromBackupResult { + created: number + updated: number + deleted: number + errors: string[] +} + +interface ScannedAssetHash { + name: string + hash: string +} + +let preferredRelativePathByDocId = new Map() +let recentImportedRelativePaths = new Map() +let RECENT_IMPORT_WINDOW_MS = 30_000 + +async function hashContent(content: string): Promise { + let encoder = new TextEncoder() + let data = encoder.encode(content) + let hashBuffer = await crypto.subtle.digest("SHA-256", data) + let hashArray = Array.from(new Uint8Array(hashBuffer)) + return hashArray + .map(b => b.toString(16).padStart(2, "0")) + .join("") + .slice(0, 16) +} + +async function syncBackup( + handle: FileSystemDirectoryHandle, + docs: BackupDoc[], +): Promise { + await performSyncBackup(handle, docs) +} + +async function syncFromBackup( + handle: FileSystemDirectoryHandle, + targetDocs: DocumentList, + canWrite: boolean, + lastPullAtMs: number | null = null, +): Promise { + let result: SyncFromBackupResult = { + created: 0, + updated: 0, + deleted: 0, + errors: [], + } + + let manifest = await readManifest(handle) + let scannedFiles = await scanBackupFolder(handle) + let listOwner = targetDocs.$jazz.owner + + let manifestByPath = new Map( + manifest?.entries.map(e => [e.relativePath, e]) ?? [], + ) + let scannedByPath = new Map(scannedFiles.map(f => [f.relativePath, f])) + let matchedManifestDocIds = new Set() + + for (let file of scannedFiles) { + try { + let contentHash = await hashContent(file.content) + let scannedAssetHashes = await hashScannedAssets(file.assets) + let manifestEntry = manifestByPath.get(file.relativePath) + let assetsInSync = false + if (manifestEntry) { + matchedManifestDocIds.add(manifestEntry.docId) + assetsInSync = areScannedAssetsInSync( + manifestEntry.assets, + scannedAssetHashes, + ) + let unchangedSinceManifest = + manifestEntry.contentHash === contentHash && + manifestEntry.relativePath === file.relativePath && + assetsInSync + let isBeforeLastPull = + lastPullAtMs !== null && file.lastModified < lastPullAtMs + if (isBeforeLastPull && unchangedSinceManifest) continue + } + + if (!manifestEntry) { + let movedEntry = findMovedManifestEntry( + manifest, + scannedByPath, + matchedManifestDocIds, + file, + contentHash, + scannedAssetHashes, + ) + if (movedEntry) { + manifestEntry = movedEntry + matchedManifestDocIds.add(movedEntry.docId) + assetsInSync = areScannedAssetsInSync( + movedEntry.assets, + scannedAssetHashes, + ) + } + } + + if (!manifestEntry) { + if (!canWrite) { + result.errors.push(`Cannot create ${file.name}: no write permission`) + continue + } + if (wasRecentlyImported(file.relativePath)) continue + let newDocId = await createDocFromFile(file, targetDocs, listOwner) + preferredRelativePathByDocId.set(newDocId, file.relativePath) + markRecentlyImported(file.relativePath) + result.created++ + } else if ( + manifestEntry.contentHash !== contentHash || + manifestEntry.relativePath !== file.relativePath || + !assetsInSync + ) { + preferredRelativePathByDocId.set(manifestEntry.docId, file.relativePath) + if (!canWrite) { + result.errors.push(`Cannot update ${file.name}: no write permission`) + continue + } + let didUpdate = await updateDocFromFile( + file, + manifestEntry.docId, + manifestEntry, + targetDocs, + ) + if (didUpdate) { + result.updated++ + } else { + result.errors.push( + `Skipped update for ${file.relativePath}: target document not loaded`, + ) + } + } + } catch (err) { + result.errors.push( + `Failed to process ${file.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, + ) + } + } + + if (manifest && canWrite) { + for (let entry of manifest.entries) { + if (matchedManifestDocIds.has(entry.docId)) continue + if (!scannedByPath.has(entry.relativePath)) { + try { + let doc = targetDocs.find(d => d?.$jazz.id === entry.docId) + if (doc?.$isLoaded && !doc.deletedAt) { + doc.$jazz.set("deletedAt", new Date()) + doc.$jazz.set("updatedAt", new Date()) + result.deleted++ + } + } catch (err) { + result.errors.push( + `Failed to delete ${entry.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, + ) + } + } + } + } + + return result +} + +async function prepareBackupDoc(doc: LoadedDocument): Promise { + let content = doc.content?.toString() ?? "" + let title = getDocumentTitle(doc) + let path = getPath(content) + let updatedAtMs = doc.updatedAt?.getTime() ?? 0 + + let assets: BackupDoc["assets"] = [] + if (doc.assets?.$isLoaded) { + for (let asset of [...doc.assets]) { + if (!asset?.$isLoaded) continue + + let blob: Blob | undefined + if (asset.type === "image" && asset.image?.$isLoaded) { + let original = asset.image.original + if (original?.$isLoaded) { + blob = original.toBlob() + } + } else if (asset.type === "video" && asset.video?.$isLoaded) { + blob = await asset.video.toBlob() + } + + if (blob) { + assets.push({ id: asset.$jazz.id, name: asset.name, blob }) + } + } + } + + return { id: doc.$jazz.id, title, content, path, updatedAtMs, assets } +} + +async function getOrCreateDirectory( + parent: FileSystemDirectoryHandle, + path: string, +): Promise { + let parts = path.split("/").filter(Boolean) + let current = parent + for (let part of parts) { + current = await current.getDirectoryHandle(part, { create: true }) + } + return current +} + +async function writeFile( + dir: FileSystemDirectoryHandle, + name: string, + content: string | Blob, +): Promise { + let file = await dir.getFileHandle(name, { create: true }) + let writable = await file.createWritable() + await writable.write(content) + await writable.close() +} + +async function deleteFile( + dir: FileSystemDirectoryHandle, + name: string, +): Promise { + try { + await dir.removeEntry(name) + } catch { + return + } +} + +async function listFiles(dir: FileSystemDirectoryHandle): Promise { + let files: string[] = [] + for await (let [name, handle] of dir.entries()) { + if (handle.kind === "file") files.push(name) + } + return files +} + +async function listDirectories( + dir: FileSystemDirectoryHandle, +): Promise { + let dirs: string[] = [] + for await (let [name, handle] of dir.entries()) { + if (handle.kind === "directory") dirs.push(name) + } + return dirs +} + +async function performSyncBackup( + handle: FileSystemDirectoryHandle, + docs: BackupDoc[], +): Promise { + let docLocations = computeDocLocations(docs) + let existingManifest = await readManifest(handle) + let existingEntriesByDocId = new Map( + existingManifest?.entries.map(entry => [entry.docId, entry]) ?? [], + ) + let manifestEntries: { + docId: string + relativePath: string + locationKey: string + contentHash: string + lastSyncedAt: string + assets: { id: string; name: string; hash: string }[] + }[] = [] + let hasFilesystemChanges = false + let nowIso = new Date().toISOString() + + for (let doc of docs) { + let loc = docLocations.get(doc.id)! + let locationKey = getDocLocationKey(doc) + let computedRelativePath = loc.dirPath + ? `${loc.dirPath}/${loc.filename}` + : loc.filename + let existingEntry = existingEntriesByDocId.get(doc.id) + let preferredRelativePath = preferredRelativePathByDocId.get(doc.id) + let finalRelativePath = computedRelativePath + if (existingEntry) { + if (existingEntry.locationKey === locationKey) { + finalRelativePath = existingEntry.relativePath + } + } + if (preferredRelativePath) { + finalRelativePath = preferredRelativePath + } + let finalLocation = buildLocationFromRelativePath(loc, finalRelativePath) + docLocations.set(doc.id, finalLocation) + loc = finalLocation + + let dir = loc.dirPath + ? await getOrCreateDirectory(handle, loc.dirPath) + : handle + + let exportedContent = transformContentForBackup(doc.content, loc.assetFiles) + let contentHash = await hashContent(exportedContent) + let relativePath = finalRelativePath + let assets: { id: string; name: string; hash: string }[] = [] + for (let asset of doc.assets) { + let filename = loc.assetFiles.get(asset.id)! + assets.push({ + id: asset.id, + name: filename, + hash: await hashBlob(asset.blob), + }) + } + + let shouldWriteDoc = + !existingEntry || + existingEntry.relativePath !== relativePath || + existingEntry.contentHash !== contentHash || + !areManifestAssetsEqual(existingEntry.assets, assets) + + if (shouldWriteDoc) { + hasFilesystemChanges = true + await writeFile(dir, loc.filename, exportedContent) + + if (doc.assets.length > 0) { + let assetsDir = await dir.getDirectoryHandle("assets", { create: true }) + for (let asset of doc.assets) { + let filename = loc.assetFiles.get(asset.id)! + await writeFile(assetsDir, filename, asset.blob) + } + } + } + + manifestEntries.push({ + docId: doc.id, + relativePath, + locationKey, + contentHash, + lastSyncedAt: shouldWriteDoc + ? nowIso + : (existingEntry?.lastSyncedAt ?? nowIso), + assets, + }) + } + + let docsChanged = + existingEntriesByDocId.size !== manifestEntries.length || + hasFilesystemChanges + + if (docsChanged) { + await cleanupOrphanedFiles(handle, docs, docLocations) + + await writeManifest(handle, { + version: 1, + entries: manifestEntries, + lastSyncAt: nowIso, + }) + + for (let entry of manifestEntries) { + recentImportedRelativePaths.delete(entry.relativePath) + } + } + + for (let doc of docs) { + preferredRelativePathByDocId.delete(doc.id) + } +} + +async function cleanupOrphanedFiles( + handle: FileSystemDirectoryHandle, + docs: BackupDoc[], + docLocations: Map< + string, + ReturnType extends Map + ? V + : never + >, +): Promise { + let { expectedPaths, expectedFiles } = computeExpectedStructure( + docs, + docLocations, + ) + let expectedAssetFilesByDir = new Map>() + for (let doc of docs) { + let location = docLocations.get(doc.id) + if (!location || !location.hasOwnFolder) continue + let assetsPath = location.dirPath ? `${location.dirPath}/assets` : "assets" + expectedAssetFilesByDir.set( + assetsPath, + new Set(location.assetFiles.values()), + ) + } + + async function cleanAssetsDirectory( + dir: FileSystemDirectoryHandle, + path: string, + ): Promise { + let hasContent = false + let expected = expectedAssetFilesByDir.get(path) ?? new Set() + + for await (let [name, child] of dir.entries()) { + if (name.startsWith(".")) continue + if (child.kind === "directory") { + await dir.removeEntry(name, { recursive: true }) + continue + } + if (expected.has(name)) { + hasContent = true + continue + } + await deleteFile(dir, name) + } + + return hasContent + } + + async function cleanDir( + dir: FileSystemDirectoryHandle, + path: string, + ): Promise { + let subdirs = await listDirectories(dir) + let hasContent = false + + for (let subdir of subdirs) { + let subPath = path ? `${path}/${subdir}` : subdir + + if (subdir === "assets" && expectedPaths.has(subPath)) { + let subHandle = await dir.getDirectoryHandle(subdir) + let assetsHasContent = await cleanAssetsDirectory(subHandle, subPath) + if (assetsHasContent) hasContent = true + continue + } + + if (expectedPaths.has(subPath)) { + let subHandle = await dir.getDirectoryHandle(subdir) + let subHasContent = await cleanDir(subHandle, subPath) + if (subHasContent) hasContent = true + } else { + try { + await dir.removeEntry(subdir, { recursive: true }) + } catch { + continue + } + } + } + + let expected = expectedFiles.get(path) ?? new Set() + let files = await listFiles(dir) + for (let file of files) { + if (file.endsWith(".md")) { + if (expected.has(file)) { + hasContent = true + } else { + await deleteFile(dir, file) + } + } + } + + return hasContent + } + + await cleanDir(handle, "") +} + +async function createDocFromFile( + file: ScannedFile, + targetDocs: DocumentList, + listOwner: Group | Account, +): Promise { + let docGroup = Group.create() + if (listOwner instanceof Group) { + docGroup.addMember(listOwner) + } + + let now = new Date() + + let docAssets: co.loaded[] = [] + let assetFilesById = new Map() + for (let assetFile of file.assets) { + let asset = await createAssetFromBlob(assetFile, docGroup, now) + docAssets.push(asset) + assetFilesById.set(asset.$jazz.id, assetFile.name) + } + + let transformedContent = transformContentForImport( + file.content, + assetFilesById, + ) + let content = applyPathFromRelativePath( + transformedContent, + file.relativePath, + file.assets.length > 0, + ) + + let newDoc = Document.create( + { + version: 1, + content: co.plainText().create(content, docGroup), + assets: + docAssets.length > 0 + ? co.list(Asset).create(docAssets, docGroup) + : undefined, + createdAt: now, + updatedAt: now, + }, + docGroup, + ) + + targetDocs.$jazz.push(newDoc) + return newDoc.$jazz.id +} + +async function updateDocFromFile( + file: ScannedFile, + docId: string, + manifestEntry: ManifestEntry, + targetDocs: DocumentList, +): Promise { + let doc = targetDocs.find( + (d): d is LoadedDocument => d?.$isLoaded === true && d.$jazz.id === docId, + ) + if (!doc || !doc.content?.$isLoaded) { + return false + } + + let assetFilesById = await syncDocAssetsFromFile(doc, file, manifestEntry) + let content = applyPathFromRelativePath( + transformContentForImport(file.content, assetFilesById), + file.relativePath, + file.assets.length > 0, + ) + + doc.content.$jazz.applyDiff(content) + doc.$jazz.set("updatedAt", new Date()) + return true +} + +async function syncDocAssetsFromFile( + doc: LoadedDocument, + file: ScannedFile, + manifestEntry: ManifestEntry, +): Promise> { + if (file.assets.length === 0) { + let manifestFilesById = getAssetFilesByIdFromManifest(manifestEntry) + let keepsManifestRefs = Array.from(manifestFilesById.values()).some( + filename => file.content.includes(`assets/${filename}`), + ) + if (keepsManifestRefs) { + return manifestFilesById + } + if (doc.assets?.$isLoaded) { + for (let i = doc.assets.length - 1; i >= 0; i--) { + doc.assets.$jazz.splice(i, 1) + } + } + return new Map() + } + + if (!doc.assets) { + doc.$jazz.set("assets", co.list(Asset).create([], doc.$jazz.owner)) + } + if (!doc.assets?.$isLoaded) { + return getAssetFilesByIdFromManifest(manifestEntry) + } + + let currentAssets = Array.from(doc.assets).filter( + (asset): asset is co.loaded => asset?.$isLoaded === true, + ) + let assetsById = new Map(currentAssets.map(asset => [asset.$jazz.id, asset])) + let fileAssetsWithHash = await Promise.all( + file.assets.map(async asset => ({ + name: asset.name, + blob: asset.blob, + hash: await hashBlob(asset.blob), + })), + ) + + let manifestByName = new Map( + manifestEntry.assets.map(asset => [asset.name, asset]), + ) + let manifestByHash = new Map() + for (let asset of manifestEntry.assets) { + if (!manifestByHash.has(asset.hash)) { + manifestByHash.set(asset.hash, []) + } + manifestByHash.get(asset.hash)?.push(asset) + } + + let keepAssetIds = new Set() + let assetFilesById = new Map() + + for (let fileAsset of fileAssetsWithHash) { + let matchedByName = manifestByName.get(fileAsset.name) + let matchedId = + matchedByName?.id && assetsById.has(matchedByName.id) + ? matchedByName.id + : null + + if (!matchedId) { + let byHash = manifestByHash.get(fileAsset.hash) ?? [] + for (let candidate of byHash) { + if (!candidate.id || keepAssetIds.has(candidate.id)) continue + if (!assetsById.has(candidate.id)) continue + matchedId = candidate.id + break + } + } + + if (matchedId) { + let existing = assetsById.get(matchedId) + if (!existing) continue + + let shouldUpdateBinary = + matchedByName?.id === matchedId + ? matchedByName.hash !== fileAsset.hash + : false + + if (shouldUpdateBinary) { + matchedId = await syncExistingAssetFromFile( + doc, + existing, + matchedId, + fileAsset, + ) + } + + let updatedAsset = doc.assets.find( + asset => asset?.$isLoaded && asset.$jazz.id === matchedId, + ) + if (!updatedAsset) continue + assetsById.set(matchedId, updatedAsset) + if (updatedAsset.name !== removeExtension(fileAsset.name)) { + updatedAsset.$jazz.applyDiff({ name: removeExtension(fileAsset.name) }) + } + + keepAssetIds.add(matchedId) + assetFilesById.set(matchedId, fileAsset.name) + continue + } + + let created = await createAssetFromBlob( + fileAsset, + doc.$jazz.owner, + new Date(), + ) + doc.assets.$jazz.push(created) + keepAssetIds.add(created.$jazz.id) + assetFilesById.set(created.$jazz.id, fileAsset.name) + } + + for (let i = doc.assets.length - 1; i >= 0; i--) { + let asset = doc.assets[i] + if (!asset?.$isLoaded) continue + if (keepAssetIds.has(asset.$jazz.id)) continue + doc.assets.$jazz.splice(i, 1) + } + + if (fileAssetsWithHash.length === 0) { + return new Map() + } + + return assetFilesById +} + +async function syncExistingAssetFromFile( + doc: LoadedDocument, + existing: co.loaded, + assetId: string, + fileAsset: { name: string; blob: Blob }, +): Promise { + let nextType = fileAsset.blob.type.startsWith("video/") ? "video" : "image" + if (existing.type !== nextType) { + let index = + doc.assets?.findIndex(asset => asset?.$jazz.id === assetId) ?? -1 + if (index === -1 || !doc.assets) return assetId + doc.assets.$jazz.splice(index, 1) + let replacement = await createAssetFromBlob( + fileAsset, + doc.$jazz.owner, + existing.createdAt, + ) + doc.assets.$jazz.push(replacement) + return replacement.$jazz.id + } + + if (existing.type === "video") { + let stream = await FileStream.createFromBlob(fileAsset.blob, { + owner: doc.$jazz.owner, + }) + existing.$jazz.applyDiff({ + name: removeExtension(fileAsset.name), + video: stream, + mimeType: fileAsset.blob.type || "video/mp4", + }) + return assetId + } + + let image = await createImage(fileAsset.blob, { + owner: doc.$jazz.owner, + maxSize: 2048, + }) + existing.$jazz.applyDiff({ name: removeExtension(fileAsset.name), image }) + return assetId +} + +async function createAssetFromBlob( + assetFile: { name: string; blob: Blob }, + owner: Group, + now: Date, +) { + let isVideo = assetFile.blob.type.startsWith("video/") + if (isVideo) { + let video = await FileStream.createFromBlob(assetFile.blob, { + owner, + }) + return VideoAsset.create( + { + type: "video", + name: removeExtension(assetFile.name), + video, + mimeType: assetFile.blob.type || "video/mp4", + createdAt: now, + }, + owner, + ) + } + + let image = await createImage(assetFile.blob, { + owner, + maxSize: 2048, + }) + return ImageAsset.create( + { + type: "image", + name: removeExtension(assetFile.name), + image, + createdAt: now, + }, + owner, + ) +} + +function removeExtension(filename: string): string { + return filename.replace(/\.[^.]+$/, "") +} + +function findMovedManifestEntry( + manifest: Awaited>, + scannedByPath: Map, + matchedManifestDocIds: Set, + file: ScannedFile, + contentHash: string, + scannedAssetHashes: ScannedAssetHash[], +) { + if (!manifest) return null + let candidates = manifest.entries.filter(entry => { + if (matchedManifestDocIds.has(entry.docId)) return false + if (scannedByPath.has(entry.relativePath)) return false + if (entry.contentHash !== contentHash) return false + if (!areScannedAssetsInSync(entry.assets, scannedAssetHashes)) return false + return true + }) + if (candidates.length === 0) return null + if (candidates.length === 1) return candidates[0] + + let matchingBasename = candidates.filter(entry => { + return getFilename(entry.relativePath) === getFilename(file.relativePath) + }) + if (matchingBasename.length === 1) { + return matchingBasename[0] + } + + return null +} + +function getFilename(relativePath: string): string { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return relativePath + return parts[parts.length - 1] +} + +function getAssetFilesByIdFromManifest( + manifestEntry: ManifestEntry, +): Map { + let filesById = new Map() + for (let asset of manifestEntry.assets) { + if (!asset.id) continue + filesById.set(asset.id, asset.name) + } + return filesById +} + +function applyPathFromRelativePath( + content: string, + relativePath: string, + hasAssets: boolean, +): string { + let diskPath = derivePathFromRelativePath(relativePath, hasAssets) + let { frontmatter } = parseFrontmatter(content) + let currentPath = getPath(content) + + if (!frontmatter) { + if (!diskPath) return content + return `---\npath: ${diskPath}\n---\n\n${content}` + } + + if (currentPath === diskPath) return content + + if (currentPath && !diskPath) { + return content.replace( + /^(---\r?\n[\s\S]*?)path:\s*[^\r\n]*\r?\n([\s\S]*?---)/, + "$1$2", + ) + } + + if (currentPath && diskPath) { + return content.replace( + /^(---\r?\n[\s\S]*?)path:\s*[^\r\n]*/, + `$1path: ${diskPath}`, + ) + } + + if (!currentPath && diskPath) { + return content.replace(/^(---\r?\n)/, `$1path: ${diskPath}\n`) + } + + return content +} + +function derivePathFromRelativePath( + relativePath: string, + hasAssets: boolean, +): string | null { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length <= 1) return null + + let directoryParts = parts.slice(0, -1) + if (!hasAssets) { + let path = directoryParts.join("/") + return path || null + } + + let parentParts = directoryParts.slice(0, -1) + let path = parentParts.join("/") + return path || null +} + +function buildLocationFromRelativePath( + baseLocation: ReturnType extends Map< + string, + infer V + > + ? V + : never, + relativePath: string, +) { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return baseLocation + + return { + ...baseLocation, + dirPath: parts.slice(0, -1).join("/"), + filename: parts[parts.length - 1], + } +} + +async function hashBlob(blob: Blob): Promise { + let buffer = await blob.arrayBuffer() + let hashBuffer = await crypto.subtle.digest("SHA-256", buffer) + let hashArray = Array.from(new Uint8Array(hashBuffer)) + return hashArray + .map(b => b.toString(16).padStart(2, "0")) + .join("") + .slice(0, 16) +} + +async function hashScannedAssets( + assets: { name: string; blob: Blob }[], +): Promise { + let hashed = await Promise.all( + assets.map(async asset => ({ + name: asset.name, + hash: await hashBlob(asset.blob), + })), + ) + return hashed +} + +function areScannedAssetsInSync( + manifestAssets: { id?: string; name: string; hash: string }[], + scannedAssets: ScannedAssetHash[], +): boolean { + if (manifestAssets.length !== scannedAssets.length) return false + + let sortedManifest = [...manifestAssets].sort((a, b) => + a.name.localeCompare(b.name), + ) + let sortedScanned = [...scannedAssets].sort((a, b) => + a.name.localeCompare(b.name), + ) + + for (let i = 0; i < sortedManifest.length; i++) { + if (sortedManifest[i].name !== sortedScanned[i].name) return false + if (sortedManifest[i].hash !== sortedScanned[i].hash) return false + } + + return true +} + +function areManifestAssetsEqual( + a: { id?: string; name: string; hash: string }[], + b: { id?: string; name: string; hash: string }[], +): boolean { + if (a.length !== b.length) return false + + let sortedA = [...a].sort((left, right) => + left.name.localeCompare(right.name), + ) + let sortedB = [...b].sort((left, right) => + left.name.localeCompare(right.name), + ) + + for (let i = 0; i < sortedA.length; i++) { + if ((sortedA[i].id ?? null) !== (sortedB[i].id ?? null)) return false + if (sortedA[i].name !== sortedB[i].name) return false + if (sortedA[i].hash !== sortedB[i].hash) return false + } + + return true +} + +function wasRecentlyImported(relativePath: string): boolean { + let importedAt = recentImportedRelativePaths.get(relativePath) + if (!importedAt) return false + if (Date.now() - importedAt > RECENT_IMPORT_WINDOW_MS) { + recentImportedRelativePaths.delete(relativePath) + return false + } + return true +} + +function markRecentlyImported(relativePath: string): void { + recentImportedRelativePaths.set(relativePath, Date.now()) +} + +function getDocLocationKey(doc: BackupDoc): string { + let path = doc.path ?? "" + let hasAssets = doc.assets.length > 0 ? "assets" : "no-assets" + return `${doc.title}|${path}|${hasAssets}` +} diff --git a/src/lib/backup-scenarios.test.ts b/src/lib/backup-scenarios.test.ts index 38a0b8b..a41601b 100644 --- a/src/lib/backup-scenarios.test.ts +++ b/src/lib/backup-scenarios.test.ts @@ -6,9 +6,10 @@ import { getPath } from "@/editor/frontmatter" import { getDocumentTitle } from "@/lib/document-utils" import type { BackupDoc } from "./backup-sync" import { readManifest } from "./backup-sync" -import { hashContent, syncBackup, syncFromBackup } from "./backup" +import { hashContent, syncBackup, syncFromBackup } from "./backup-engine" import { MockDirectoryHandle, + createMockBlob, basename, readFileAtPath as readFile, removeFileAtPath as removeFile, @@ -154,6 +155,21 @@ describe("backup scenarios", () => { expect(getPath(loaded?.content?.toString() ?? "")).toBe("archive") }) + it("imports file updates when lastModified equals last pull timestamp", async () => { + let doc = await createDoc(docs, "# Timestamp Edge\n\noriginal") + await pushToBackup(root, docs) + + let updatedContent = "# Timestamp Edge\n\nchanged on disk" + root.addFile("Timestamp Edge.md", updatedContent, 10_000) + + let result = await syncFromBackup(root, docs, true, 10_000) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + expect(loaded?.content?.toString()).toContain("changed on disk") + }) + it("matches moved doc by filename when content hashes collide", async () => { let first = await createDoc(docs, "# Same") let second = await createDoc(docs, "# Same") @@ -192,6 +208,83 @@ describe("backup scenarios", () => { expect(secondLoaded?.deletedAt).toBeFalsy() }) + it("matches moved docs with colliding hashes by asset hashes", async () => { + let first = await createDoc(docs, "# Same") + let second = await createDoc(docs, "# Same") + + let backupDocs: BackupDoc[] = [ + { + id: first.$jazz.id, + title: "Same", + content: "# Same\n\n![Clip](asset:asset-1)", + path: "alpha", + updatedAtMs: Date.now(), + assets: [ + { + id: "asset-1", + name: "clip", + blob: createMockBlob("first-video", "video/mp4"), + }, + ], + }, + { + id: second.$jazz.id, + title: "Same", + content: "# Same\n\n![Clip](asset:asset-2)", + path: "beta", + updatedAtMs: Date.now(), + assets: [ + { + id: "asset-2", + name: "clip", + blob: createMockBlob("second-video", "video/mp4"), + }, + ], + }, + ] + + await syncBackup(root, backupDocs) + + let firstOriginalPath = "alpha/Same/Same.md" + let secondOriginalPath = "beta/Same/Same.md" + let firstContent = await readFile(root, firstOriginalPath) + let secondContent = await readFile(root, secondOriginalPath) + await removeFile(root, firstOriginalPath) + await removeFile(root, secondOriginalPath) + + await writeFileAtPath(root, "archive/one/Same/Same.md", firstContent) + await writeFileAtPath( + root, + "archive/one/Same/assets/clip.mp4", + "first-video", + ) + await writeFileAtPath(root, "archive/two/Same/Same.md", secondContent) + await writeFileAtPath( + root, + "archive/two/Same/assets/clip.mp4", + "second-video", + ) + + let beforePullCount = getLoadedDocs(docs).length + let result = await syncFromBackup(root, docs, true) + + expect(result.created).toBe(0) + expect(result.updated).toBe(2) + expect(result.deleted).toBe(0) + expect(getLoadedDocs(docs)).toHaveLength(beforePullCount) + + let firstLoaded = getLoadedDocs(docs).find( + d => d.$jazz.id === first.$jazz.id, + ) + let secondLoaded = getLoadedDocs(docs).find( + d => d.$jazz.id === second.$jazz.id, + ) + expect(firstLoaded?.deletedAt).toBeFalsy() + expect(secondLoaded?.deletedAt).toBeFalsy() + expect(getPath(firstLoaded?.content?.toString() ?? "")).toBe("archive/one") + expect(getPath(secondLoaded?.content?.toString() ?? "")).toBe("archive/two") + }) + it("keeps asset refs stable when pulling updates for docs with assets", async () => { let { doc, assetId } = await createDocWithVideoAsset(docs, "Asset Sync") let localContent = @@ -234,6 +327,17 @@ describe("backup scenarios", () => { it("imports asset binary updates when markdown is unchanged", async () => { let { doc, assetId } = await createDocWithVideoAsset(docs, "Binary Update") + let originalAsset = doc.assets?.$isLoaded + ? [...doc.assets].find( + asset => asset?.$isLoaded && asset.$jazz.id === assetId, + ) + : undefined + let originalVideoId = + originalAsset?.$isLoaded && + originalAsset.type === "video" && + originalAsset.video?.$isLoaded + ? originalAsset.video.$jazz.id + : null let localPath = "Binary Update/Binary Update.md" let localContent = "# Binary Update\n\n![Clip](assets/clip.mp4)" await writeFileAtPath(root, localPath, localContent) @@ -292,9 +396,89 @@ describe("backup scenarios", () => { throw new Error("Updated video asset not loaded") } - let blob = await updatedAsset.video.toBlob() - if (!blob) throw new Error("Updated video blob missing") - expect(await blob.text()).toBe("new-video-bytes") + expect(updatedAsset.video.$jazz.id).not.toBe(originalVideoId) + }) + + it("imports asset-only updates when markdown file is older than last pull", async () => { + let { doc, assetId } = await createDocWithVideoAsset( + docs, + "Asset Timestamp", + ) + let originalAsset = doc.assets?.$isLoaded + ? [...doc.assets].find( + asset => asset?.$isLoaded && asset.$jazz.id === assetId, + ) + : undefined + let originalVideoId = + originalAsset?.$isLoaded && + originalAsset.type === "video" && + originalAsset.video?.$isLoaded + ? originalAsset.video.$jazz.id + : null + + let docDir = new MockDirectoryHandle("Asset Timestamp") + let assetsDir = new MockDirectoryHandle("assets") + docDir.addDirectory("assets", assetsDir) + docDir.addFile( + "Asset Timestamp.md", + "# Asset Timestamp\n\n![Clip](assets/clip.mp4)", + 1_000, + ) + assetsDir.addFile("clip.mp4", "new-video-bytes", 9_000) + root.addDirectory("Asset Timestamp", docDir) + + let contentHash = await hashContent( + "# Asset Timestamp\n\n![Clip](assets/clip.mp4)", + ) + let previousAssetHash = await hashContent("old-video-bytes") + await writeFileAtPath( + root, + ".alkalye-manifest.json", + JSON.stringify( + { + version: 1, + entries: [ + { + docId: doc.$jazz.id, + relativePath: "Asset Timestamp/Asset Timestamp.md", + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: [ + { + id: assetId, + name: "clip.mp4", + hash: previousAssetHash, + }, + ], + }, + ], + lastSyncAt: new Date().toISOString(), + }, + null, + 2, + ), + ) + + let result = await syncFromBackup(root, docs, true, 5_000) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + let updatedAsset = loaded?.assets?.$isLoaded + ? [...loaded.assets].find( + asset => asset?.$isLoaded && asset.$jazz.id === assetId, + ) + : undefined + expect(updatedAsset?.$isLoaded).toBe(true) + if ( + !updatedAsset?.$isLoaded || + updatedAsset.type !== "video" || + !updatedAsset.video?.$isLoaded + ) { + throw new Error("Updated video asset not loaded") + } + + expect(updatedAsset.video.$jazz.id).not.toBe(originalVideoId) }) it("writes asset ids to manifest entries on backup", async () => { @@ -309,7 +493,7 @@ describe("backup scenarios", () => { { id: "asset-1", name: "clip", - blob: new Blob(["video"], { type: "video/mp4" }), + blob: createMockBlob("video", "video/mp4"), }, ], }, @@ -335,12 +519,12 @@ describe("backup scenarios", () => { { id: "asset-1", name: "one", - blob: new Blob(["one"], { type: "image/png" }), + blob: createMockBlob("one", "image/png"), }, { id: "asset-2", name: "two", - blob: new Blob(["two"], { type: "image/png" }), + blob: createMockBlob("two", "image/png"), }, ], }, @@ -359,7 +543,7 @@ describe("backup scenarios", () => { { id: "asset-1", name: "one", - blob: new Blob(["one"], { type: "image/png" }), + blob: createMockBlob("one", "image/png"), }, ], }, @@ -485,7 +669,7 @@ async function createDocWithVideoAsset( let group = Group.create() let now = new Date() let stream = await FileStream.createFromBlob( - new Blob(["video"], { type: "video/mp4" }), + createMockBlob("video", "video/mp4"), { owner: group, }, diff --git a/src/lib/backup-settings-ui.tsx b/src/lib/backup-settings-ui.tsx new file mode 100644 index 0000000..feb929e --- /dev/null +++ b/src/lib/backup-settings-ui.tsx @@ -0,0 +1,299 @@ +import { useState } from "react" +import { FolderOpen, AlertCircle } from "lucide-react" +import { Button } from "@/components/ui/button" +import { + useBackupStore, + enableBackup, + disableBackup, + changeBackupDirectory, + useSpaceBackupPath, + setSpaceBackupHandle, + clearSpaceBackupHandle, + supportsFileSystemWatch, + isBackupSupported, +} from "@/lib/backup-storage" + +export { BackupSettings, SpaceBackupSettings } + +function BackupSettings() { + let { + enabled, + bidirectional, + directoryName, + lastBackupAt, + lastPullAt, + lastError, + setBidirectional, + } = useBackupStore() + let [isLoading, setIsLoading] = useState(false) + + if (!isBackupSupported()) { + return ( +
+

+ Local Backup +

+
+
+ +
+

+ Local backup requires a Chromium-based browser (Chrome, Edge, + Brave, or Opera). +

+

+ Safari and Firefox do not support the File System Access API + needed for this feature. +

+
+
+
+
+ ) + } + + async function handleEnable() { + setIsLoading(true) + await enableBackup() + setIsLoading(false) + } + + async function handleDisable() { + setIsLoading(true) + await disableBackup() + setIsLoading(false) + } + + async function handleChangeDirectory() { + setIsLoading(true) + await changeBackupDirectory() + setIsLoading(false) + } + + let lastBackupDate = lastBackupAt ? new Date(lastBackupAt) : null + let formattedLastBackup = lastBackupDate + ? lastBackupDate.toLocaleString() + : null + + let lastPullDate = lastPullAt ? new Date(lastPullAt) : null + let formattedLastPull = lastPullDate ? lastPullDate.toLocaleString() : null + + return ( +
+

+ Local Backup +

+
+ {enabled ? ( + <> +
+ + + {bidirectional ? "Syncing" : "Backing up"} to folder + +
+

+ Folder: {directoryName} +

+ {formattedLastBackup && ( +

+ Last backup: {formattedLastBackup} +

+ )} + {bidirectional && formattedLastPull && ( +

+ Last sync: {formattedLastPull} +

+ )} + {lastError && ( +
+ + {lastError} +
+ )} +
+ +

+ {supportsFileSystemWatch() + ? "When enabled, changes made in the backup folder will be imported into Alkalye." + : "Requires a Chromium-based browser with File System Observer support."} +

+
+
+ + +
+ + ) : ( + <> +
+ Automatic backup disabled +
+

+ Automatically back up your documents to a folder on this device. +

+ + + )} +
+
+ ) +} + +interface SpaceBackupSettingsProps { + spaceId: string + isAdmin: boolean +} + +function SpaceBackupSettings({ spaceId, isAdmin }: SpaceBackupSettingsProps) { + let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) + let [isLoading, setIsLoading] = useState(false) + + if (!isBackupSupported()) { + return ( +
+

+ Local Backup +

+
+
+ +
+

+ Local backup requires a Chromium-based browser (Chrome, Edge, + Brave, or Opera). +

+

+ Safari and Firefox do not support the File System Access API + needed for this feature. +

+
+
+
+
+ ) + } + + async function handleChooseFolder() { + setIsLoading(true) + try { + let handle = await window.showDirectoryPicker({ mode: "readwrite" }) + await setSpaceBackupHandle(spaceId, handle) + setDirectoryName(handle.name) + } catch (e) { + if (!(e instanceof Error && e.name === "AbortError")) { + console.error("Failed to select folder:", e) + } + } finally { + setIsLoading(false) + } + } + + async function handleChangeFolder() { + await handleChooseFolder() + } + + async function handleClear() { + setIsLoading(true) + try { + await clearSpaceBackupHandle(spaceId) + setDirectoryName(null) + } finally { + setIsLoading(false) + } + } + + return ( +
+

+ Local Backup +

+
+ {directoryName ? ( + <> +
+ + Backup folder set +
+

+ Folder: {directoryName} +

+
+ + +
+ + ) : ( + <> +
+ No backup folder set +
+

+ Set a backup folder for this space's documents. +

+ + + )} +
+
+ ) +} diff --git a/src/lib/backup-storage.ts b/src/lib/backup-storage.ts new file mode 100644 index 0000000..0a1a3cb --- /dev/null +++ b/src/lib/backup-storage.ts @@ -0,0 +1,304 @@ +import { useState } from "react" +import { create } from "zustand" +import { persist } from "zustand/middleware" +import { get as idbGet, set as idbSet, del as idbDel } from "idb-keyval" + +export { + BACKUP_DEBOUNCE_MS, + SPACE_BACKUP_DEBOUNCE_MS, + SPACE_BACKUP_KEY_PREFIX, + useBackupStore, + enableBackup, + disableBackup, + changeBackupDirectory, + checkBackupPermission, + getSpaceBackupPath, + setSpaceBackupPath, + clearSpaceBackupPath, + useSpaceBackupPath, + getBackupHandle, + getSpaceBackupHandle, + setSpaceBackupHandle, + clearSpaceBackupHandle, + supportsFileSystemWatch, + isBackupSupported, + observeDirectoryChanges, + toTimestamp, +} + +declare global { + interface FileSystemObserver { + observe( + handle: FileSystemDirectoryHandle, + options?: { recursive?: boolean }, + ): Promise + disconnect(): void + } + + interface Window { + FileSystemObserver?: { + new ( + onChange: (records: unknown[], observer: FileSystemObserver) => void, + ): FileSystemObserver + } + showDirectoryPicker(options?: { + mode?: "read" | "readwrite" + }): Promise + } + + interface FileSystemDirectoryHandle { + entries(): AsyncIterableIterator<[string, FileSystemHandle]> + queryPermission(options: { + mode: "read" | "readwrite" + }): Promise<"granted" | "denied" | "prompt"> + requestPermission(options: { + mode: "read" | "readwrite" + }): Promise<"granted" | "denied" | "prompt"> + } +} + +let BACKUP_DEBOUNCE_MS = 1200 +let SPACE_BACKUP_DEBOUNCE_MS = 1200 +let HANDLE_STORAGE_KEY = "backup-directory-handle" +let SPACE_BACKUP_KEY_PREFIX = "backup-settings-space-" + +interface BackupState { + enabled: boolean + bidirectional: boolean + directoryName: string | null + lastBackupAt: string | null + lastPullAt: string | null + lastError: string | null + setEnabled: (enabled: boolean) => void + setBidirectional: (bidirectional: boolean) => void + setDirectoryName: (name: string | null) => void + setLastBackupAt: (date: string | null) => void + setLastPullAt: (date: string | null) => void + setLastError: (error: string | null) => void + reset: () => void +} + +interface SpaceBackupState { + directoryName: string | null +} + +let useBackupStore = create()( + persist( + set => ({ + enabled: false, + bidirectional: true, + directoryName: null, + lastBackupAt: null, + lastPullAt: null, + lastError: null, + setEnabled: enabled => set({ enabled }), + setBidirectional: bidirectional => set({ bidirectional }), + setDirectoryName: directoryName => set({ directoryName }), + setLastBackupAt: lastBackupAt => set({ lastBackupAt }), + setLastPullAt: lastPullAt => set({ lastPullAt }), + setLastError: lastError => set({ lastError }), + reset: () => + set({ + enabled: false, + bidirectional: true, + directoryName: null, + lastBackupAt: null, + lastPullAt: null, + lastError: null, + }), + }), + { name: "backup-settings" }, + ), +) + +async function enableBackup(): Promise<{ + success: boolean + directoryName?: string + error?: string +}> { + let handle = await requestBackupDirectory() + if (!handle) return { success: false, error: "Cancelled" } + + useBackupStore.getState().setEnabled(true) + useBackupStore.getState().setDirectoryName(handle.name) + useBackupStore.getState().setLastError(null) + + return { success: true, directoryName: handle.name } +} + +async function disableBackup(): Promise { + await clearHandle() + useBackupStore.getState().reset() +} + +async function changeBackupDirectory(): Promise<{ + success: boolean + directoryName?: string + error?: string +}> { + let handle = await requestBackupDirectory() + if (!handle) return { success: false, error: "Cancelled" } + + useBackupStore.getState().setDirectoryName(handle.name) + useBackupStore.getState().setLastError(null) + return { success: true, directoryName: handle.name } +} + +async function checkBackupPermission(): Promise { + let handle = await getBackupHandle() + return handle !== null +} + +function getSpaceBackupPath(spaceId: string): string | null { + try { + let key = getSpaceBackupStorageKey(spaceId) + let stored = localStorage.getItem(key) + if (!stored) return null + let parsed = JSON.parse(stored) + if (!isSpaceBackupState(parsed)) return null + return parsed.directoryName + } catch { + return null + } +} + +function setSpaceBackupPath(spaceId: string, directoryName: string): void { + let key = getSpaceBackupStorageKey(spaceId) + let state: SpaceBackupState = { directoryName } + localStorage.setItem(key, JSON.stringify(state)) +} + +function clearSpaceBackupPath(spaceId: string): void { + let key = getSpaceBackupStorageKey(spaceId) + localStorage.removeItem(key) +} + +function useSpaceBackupPath(spaceId: string): { + directoryName: string | null + setDirectoryName: (name: string | null) => void +} { + let [directoryName, setDirectoryNameState] = useState(() => + getSpaceBackupPath(spaceId), + ) + + function setDirectoryName(name: string | null) { + if (name) { + setSpaceBackupPath(spaceId, name) + } else { + clearSpaceBackupPath(spaceId) + } + setDirectoryNameState(name) + } + + return { directoryName, setDirectoryName } +} + +async function getBackupHandle(): Promise { + let handle = await getStoredHandle() + if (!handle) return null + let hasPermission = await verifyPermission(handle) + if (!hasPermission) return null + return handle +} + +async function getSpaceBackupHandle( + spaceId: string, +): Promise { + try { + let handle = await idbGet( + `${HANDLE_STORAGE_KEY}-space-${spaceId}`, + ) + if (!handle) return null + let hasPermission = await verifyPermission(handle) + if (!hasPermission) return null + return handle + } catch { + return null + } +} + +async function setSpaceBackupHandle( + spaceId: string, + handle: FileSystemDirectoryHandle, +): Promise { + await idbSet(`${HANDLE_STORAGE_KEY}-space-${spaceId}`, handle) +} + +async function clearSpaceBackupHandle(spaceId: string): Promise { + await idbDel(`${HANDLE_STORAGE_KEY}-space-${spaceId}`) +} + +function supportsFileSystemWatch(): boolean { + return typeof window.FileSystemObserver === "function" +} + +function isBackupSupported(): boolean { + return "showDirectoryPicker" in window +} + +async function observeDirectoryChanges( + handle: FileSystemDirectoryHandle, + onChange: () => void, +): Promise<(() => void) | null> { + let Observer = window.FileSystemObserver + if (!Observer) return null + + let observer = new Observer(() => { + onChange() + }) + await observer.observe(handle, { recursive: true }) + return () => observer.disconnect() +} + +function toTimestamp(value: string | null): number | null { + if (!value) return null + let ms = Date.parse(value) + return Number.isNaN(ms) ? null : ms +} + +function getSpaceBackupStorageKey(spaceId: string): string { + return `${SPACE_BACKUP_KEY_PREFIX}${spaceId}` +} + +function isSpaceBackupState(value: unknown): value is SpaceBackupState { + if (typeof value !== "object" || value === null) return false + if (!("directoryName" in value)) return false + return value.directoryName === null || typeof value.directoryName === "string" +} + +async function getStoredHandle(): Promise { + try { + let handle = await idbGet(HANDLE_STORAGE_KEY) + return handle ?? null + } catch { + return null + } +} + +async function storeHandle(handle: FileSystemDirectoryHandle): Promise { + await idbSet(HANDLE_STORAGE_KEY, handle) +} + +async function clearHandle(): Promise { + await idbDel(HANDLE_STORAGE_KEY) +} + +async function verifyPermission( + handle: FileSystemDirectoryHandle, +): Promise { + let opts: { mode: "readwrite" } = { mode: "readwrite" } + if ((await handle.queryPermission(opts)) === "granted") return true + if ((await handle.requestPermission(opts)) === "granted") return true + return false +} + +async function requestBackupDirectory(): Promise { + try { + let handle = await window.showDirectoryPicker({ mode: "readwrite" }) + await storeHandle(handle) + return handle + } catch (e) { + if (e instanceof Error && e.name === "AbortError") return null + throw e + } +} diff --git a/src/lib/backup-subscribers.tsx b/src/lib/backup-subscribers.tsx new file mode 100644 index 0000000..c4de442 --- /dev/null +++ b/src/lib/backup-subscribers.tsx @@ -0,0 +1,346 @@ +import { useEffect, useRef, useState } from "react" +import { useAccount, useCoState } from "jazz-tools/react" +import { type ResolveQuery, Group } from "jazz-tools" +import { UserAccount, Space } from "@/schema" +import { + syncBackup, + syncFromBackup, + prepareBackupDoc, + type LoadedDocument, +} from "@/lib/backup-engine" +import { + BACKUP_DEBOUNCE_MS, + SPACE_BACKUP_DEBOUNCE_MS, + SPACE_BACKUP_KEY_PREFIX, + useBackupStore, + useSpaceBackupPath, + getSpaceBackupPath, + getBackupHandle, + getSpaceBackupHandle, + supportsFileSystemWatch, + observeDirectoryChanges, + toTimestamp, +} from "@/lib/backup-storage" + +export { BackupSubscriber, SpacesBackupSubscriber } + +let backupQuery = { + root: { + documents: { + $each: { content: true, assets: { $each: { image: true, video: true } } }, + $onError: "catch", + }, + }, +} as const satisfies ResolveQuery + +let spacesBackupQuery = { + root: { + spaces: true, + }, +} as const satisfies ResolveQuery + +let spaceLastPullAtById = new Map() + +function BackupSubscriber() { + let { + enabled, + bidirectional, + lastPullAt, + setLastBackupAt, + setLastPullAt, + setLastError, + setEnabled, + setDirectoryName, + } = useBackupStore() + let me = useAccount(UserAccount, { resolve: backupQuery }) + let debounceRef = useRef | null>(null) + let lastContentHashRef = useRef("") + let isPushingRef = useRef(false) + let isPullingRef = useRef(false) + + useEffect(() => { + if (!enabled || !me.$isLoaded) return + + let docs = me.root?.documents + if (!docs?.$isLoaded) return + + let activeDocs = [...docs].filter(d => d?.$isLoaded && !d.deletedAt) + let contentHash = activeDocs + .map(d => `${d.$jazz.id}:${d.updatedAt?.getTime()}`) + .sort() + .join("|") + + if (contentHash === lastContentHashRef.current) return + lastContentHashRef.current = contentHash + + if (debounceRef.current) clearTimeout(debounceRef.current) + debounceRef.current = setTimeout(async () => { + try { + let handle = await getBackupHandle() + if (!handle) { + setEnabled(false) + setDirectoryName(null) + setLastError("Permission lost - please re-enable backup") + return + } + + let loadedDocs = activeDocs.filter( + (d): d is LoadedDocument => d?.$isLoaded === true, + ) + let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) + isPushingRef.current = true + await syncBackup(handle, backupDocs) + + setLastBackupAt(new Date().toISOString()) + setLastError(null) + } catch (e) { + setLastError(e instanceof Error ? e.message : "Backup failed") + } finally { + isPushingRef.current = false + } + }, BACKUP_DEBOUNCE_MS) + + return () => { + if (debounceRef.current) clearTimeout(debounceRef.current) + } + }, [enabled, me, setLastBackupAt, setLastError, setEnabled, setDirectoryName]) + + useEffect(() => { + if (!enabled || !bidirectional || !me.$isLoaded) return + if (!supportsFileSystemWatch()) return + + let docs = me.root?.documents + if (!docs?.$isLoaded) return + + async function doPull() { + try { + if (isPushingRef.current) return + if (isPullingRef.current) return + isPullingRef.current = true + let handle = await getBackupHandle() + if (!handle) return + + if (!docs.$isLoaded || !isDocumentList(docs)) return + let result = await syncFromBackup( + handle, + docs, + true, + toTimestamp(lastPullAt), + ) + if (result.errors.length > 0) { + console.warn("Backup pull errors:", result.errors) + } + + setLastPullAt(new Date().toISOString()) + } catch (e) { + console.error("Backup pull failed:", e) + } finally { + isPullingRef.current = false + } + } + + let watchAborted = false + let stopWatching: (() => void) | null = null + + async function setupWatch() { + let handle = await getBackupHandle() + if (!handle) return + + let stop = await observeDirectoryChanges(handle, () => { + if (!watchAborted) doPull() + }) + if (watchAborted) { + stop?.() + return + } + stopWatching = stop + } + + setupWatch() + + return () => { + watchAborted = true + stopWatching?.() + } + }, [enabled, bidirectional, me, setLastPullAt, lastPullAt]) + + return null +} + +function SpacesBackupSubscriber() { + let me = useAccount(UserAccount, { resolve: spacesBackupQuery }) + let [storageVersion, setStorageVersion] = useState(0) + + useEffect(() => { + function handleStorageChange(e: StorageEvent) { + if (e.key?.startsWith(SPACE_BACKUP_KEY_PREFIX)) { + setStorageVersion(v => v + 1) + } + } + + window.addEventListener("storage", handleStorageChange) + return () => window.removeEventListener("storage", handleStorageChange) + }, []) + + let spacesWithBackup = getSpacesWithBackup(me, storageVersion) + + return ( + <> + {spacesWithBackup.map(spaceId => ( + + ))} + + ) +} + +interface SpaceBackupSubscriberProps { + spaceId: string +} + +function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { + let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) + let debounceRef = useRef | null>(null) + let lastContentHashRef = useRef("") + let isPushingRef = useRef(false) + let isPullingRef = useRef(false) + + let space = useCoState(Space, spaceId, { + resolve: { + documents: { + $each: { content: true, assets: { $each: { image: true } } }, + $onError: "catch", + }, + }, + }) + + useEffect(() => { + if (!directoryName) return + if (!space?.$isLoaded || !space.documents?.$isLoaded) return + + let docs = space.documents + let activeDocs = [...docs].filter(d => d?.$isLoaded && !d.deletedAt) + + let contentHash = activeDocs + .map(d => `${d.$jazz.id}:${d.updatedAt?.getTime()}`) + .sort() + .join("|") + + if (contentHash === lastContentHashRef.current) return + lastContentHashRef.current = contentHash + + if (debounceRef.current) clearTimeout(debounceRef.current) + debounceRef.current = setTimeout(async () => { + try { + let handle = await getSpaceBackupHandle(spaceId) + if (!handle) { + setDirectoryName(null) + return + } + + let loadedDocs = activeDocs.filter( + (d): d is LoadedDocument => d?.$isLoaded === true, + ) + let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) + isPushingRef.current = true + await syncBackup(handle, backupDocs) + } catch (e) { + console.error(`Space backup failed for ${spaceId}:`, e) + } finally { + isPushingRef.current = false + } + }, SPACE_BACKUP_DEBOUNCE_MS) + + return () => { + if (debounceRef.current) clearTimeout(debounceRef.current) + } + }, [directoryName, space, spaceId, setDirectoryName]) + + useEffect(() => { + if (!directoryName) return + if (!space?.$isLoaded || !space.documents?.$isLoaded) return + if (!supportsFileSystemWatch()) return + + let docs = space.documents + + let spaceGroup = + space.$jazz.owner instanceof Group ? space.$jazz.owner : null + let canWrite = + spaceGroup?.myRole() === "admin" || spaceGroup?.myRole() === "writer" + + async function doPull() { + try { + if (isPushingRef.current) return + if (isPullingRef.current) return + isPullingRef.current = true + let handle = await getSpaceBackupHandle(spaceId) + if (!handle) return + + if (!docs.$isLoaded || !isDocumentList(docs)) return + let result = await syncFromBackup( + handle, + docs, + canWrite, + spaceLastPullAtById.get(spaceId) ?? null, + ) + if (result.errors.length > 0) { + console.warn(`Space ${spaceId} pull errors:`, result.errors) + } + spaceLastPullAtById.set(spaceId, Date.now()) + } catch (e) { + console.error(`Space backup pull failed for ${spaceId}:`, e) + } finally { + isPullingRef.current = false + } + } + + let watchAborted = false + let stopWatching: (() => void) | null = null + + async function setupWatch() { + let handle = await getSpaceBackupHandle(spaceId) + if (!handle) return + + let stop = await observeDirectoryChanges(handle, () => { + if (!watchAborted) doPull() + }) + if (watchAborted) { + stop?.() + return + } + stopWatching = stop + } + + setupWatch() + + return () => { + watchAborted = true + stopWatching?.() + } + }, [directoryName, space, spaceId]) + + return null +} + +function getSpacesWithBackup( + me: ReturnType< + typeof useAccount + >, + _storageVersion: number, +): string[] { + if (!me.$isLoaded || !me.root?.spaces?.$isLoaded) return [] + + let spaceIds: string[] = [] + for (let space of Array.from(me.root.spaces)) { + if (!space?.$isLoaded) continue + let backupPath = getSpaceBackupPath(space.$jazz.id) + if (backupPath) { + spaceIds.push(space.$jazz.id) + } + } + return spaceIds +} + +function isDocumentList(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false + return "$jazz" in value && "find" in value +} diff --git a/src/lib/backup-sync.test.ts b/src/lib/backup-sync.test.ts index 3cb81e5..be1c1d5 100644 --- a/src/lib/backup-sync.test.ts +++ b/src/lib/backup-sync.test.ts @@ -108,6 +108,66 @@ describe("computeDocLocations", () => { expect(loc?.assetFiles.get("a2")).toBeDefined() expect(loc?.assetFiles.get("a1")).not.toBe(loc?.assetFiles.get("a2")) }) + + it("assigns stable duplicate asset filenames regardless of asset order", () => { + let docForward = createDoc({ + id: "d1", + title: "Assets", + path: null, + assets: [ + { id: "a1", name: "shot", blob: createMockBlob("one") }, + { id: "a2", name: "shot", blob: createMockBlob("two") }, + ], + }) + let docReverse = createDoc({ + id: "d1", + title: "Assets", + path: null, + assets: [ + { id: "a2", name: "shot", blob: createMockBlob("two") }, + { id: "a1", name: "shot", blob: createMockBlob("one") }, + ], + }) + + let forwardLoc = computeDocLocations([docForward]).get("d1") + let reverseLoc = computeDocLocations([docReverse]).get("d1") + + expect(forwardLoc?.assetFiles.get("a1")).toBe( + reverseLoc?.assetFiles.get("a1"), + ) + expect(forwardLoc?.assetFiles.get("a2")).toBe( + reverseLoc?.assetFiles.get("a2"), + ) + }) + + it("assigns stable collision filenames regardless of input order", () => { + let first: BackupDoc = { + id: "doc-aaaa1111", + title: "Same", + content: "x", + path: "work", + updatedAtMs: 0, + assets: [], + } + let second: BackupDoc = { + id: "doc-bbbb2222", + title: "Same", + content: "x", + path: "work", + updatedAtMs: 0, + assets: [], + } + + let forward = computeDocLocations([first, second]) + let reverse = computeDocLocations([second, first]) + + expect(forward.get(first.id)?.filename).toBe( + reverse.get(first.id)?.filename, + ) + expect(forward.get(second.id)?.filename).toBe( + reverse.get(second.id)?.filename, + ) + }) }) // ============================================================================= @@ -261,10 +321,7 @@ describe("readManifest", () => { lastSyncAt: new Date().toISOString(), } - root.addFile( - ".alkalye-manifest.json", - new File([JSON.stringify(manifest)], ".alkalye-manifest.json"), - ) + root.addFile(".alkalye-manifest.json", JSON.stringify(manifest)) let result = await readManifest(root) @@ -278,10 +335,7 @@ describe("readManifest", () => { let root = new MockDirectoryHandle("root") let invalidManifest = { version: 2, entries: [] } - root.addFile( - ".alkalye-manifest.json", - new File([JSON.stringify(invalidManifest)], ".alkalye-manifest.json"), - ) + root.addFile(".alkalye-manifest.json", JSON.stringify(invalidManifest)) let result = await readManifest(root) @@ -296,10 +350,7 @@ describe("readManifest", () => { lastSyncAt: new Date().toISOString(), } - root.addFile( - ".alkalye-manifest.json", - new File([JSON.stringify(invalidManifest)], ".alkalye-manifest.json"), - ) + root.addFile(".alkalye-manifest.json", JSON.stringify(invalidManifest)) let result = await readManifest(root) @@ -308,10 +359,7 @@ describe("readManifest", () => { it("returns null for malformed JSON", async () => { let root = new MockDirectoryHandle("root") - root.addFile( - ".alkalye-manifest.json", - new File(["{invalid"], ".alkalye-manifest.json"), - ) + root.addFile(".alkalye-manifest.json", "{invalid") let result = await readManifest(root) @@ -379,6 +427,29 @@ describe("scanBackupFolder", () => { expect(paths).toEqual(["Root.md", "work/Work.md", "work/notes/Notes.md"]) }) + it("returns files in stable lexicographic order", async () => { + let root = new MockDirectoryHandle("root") + root.addFile("z-last.md", createMockFile("# z")) + root.addFile("a-first.md", createMockFile("# a")) + + let files = await scanBackupFolder(root) + + expect(files.map(file => file.relativePath)).toEqual([ + "a-first.md", + "z-last.md", + ]) + }) + + it("scans markdown extension case-insensitively", async () => { + let root = new MockDirectoryHandle("root") + root.addFile("UPPER.MD", createMockFile("# Upper")) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(1) + expect(files[0].name).toBe("UPPER") + }) + it("collects assets from assets folders", async () => { let root = new MockDirectoryHandle("root") let docDir = new MockDirectoryHandle("My Doc") @@ -391,10 +462,7 @@ describe("scanBackupFolder", () => { "My Doc.md", createMockFile("# My Doc\n\n![Image](assets/photo.png)"), ) - assetsDir.addFile( - "photo.png", - new File([createMockBlob("image data")], "photo.png"), - ) + assetsDir.addFile("photo.png", "image data") let files = await scanBackupFolder(root) @@ -403,6 +471,39 @@ describe("scanBackupFolder", () => { expect(files[0].assets[0].name).toBe("photo.png") }) + it("loads only referenced assets for each markdown file", async () => { + let root = new MockDirectoryHandle("root") + root.addFile("Doc One.md", createMockFile("![One](assets/one.png)")) + root.addFile("Doc Two.md", createMockFile("No assets here")) + + let assetsDir = new MockDirectoryHandle("assets") + assetsDir.addFile("one.png", "1") + assetsDir.addFile("two.png", "2") + root.addDirectory("assets", assetsDir) + + let files = await scanBackupFolder(root) + let byName = new Map(files.map(file => [file.name, file])) + + expect(byName.get("Doc One")?.assets.map(asset => asset.name)).toEqual([ + "one.png", + ]) + expect(byName.get("Doc Two")?.assets).toHaveLength(0) + }) + + it("does not import markdown files inside assets directories", async () => { + let root = new MockDirectoryHandle("root") + let docDir = new MockDirectoryHandle("Doc") + let assetsDir = new MockDirectoryHandle("assets") + docDir.addDirectory("assets", assetsDir) + root.addDirectory("Doc", docDir) + docDir.addFile("Doc.md", createMockFile("# Doc")) + assetsDir.addFile("notes.md", createMockFile("# Should stay asset")) + + let files = await scanBackupFolder(root) + + expect(files.map(file => file.relativePath)).toEqual(["Doc/Doc.md"]) + }) + it("skips dot directories", async () => { let root = new MockDirectoryHandle("root") let hiddenDir = new MockDirectoryHandle(".hidden") @@ -419,10 +520,7 @@ describe("scanBackupFolder", () => { it("skips manifest file", async () => { let root = new MockDirectoryHandle("root") - root.addFile( - ".alkalye-manifest.json", - new File(['{"version":1}'], ".alkalye-manifest.json"), - ) + root.addFile(".alkalye-manifest.json", '{"version":1}') root.addFile("Test.md", createMockFile("# Test")) let files = await scanBackupFolder(root) @@ -431,10 +529,20 @@ describe("scanBackupFolder", () => { expect(files[0].name).toBe("Test") }) + it("skips hidden markdown files", async () => { + let root = new MockDirectoryHandle("root") + root.addFile(".hidden.md", createMockFile("# Hidden")) + root.addFile("Visible.md", createMockFile("# Visible")) + + let files = await scanBackupFolder(root) + + expect(files.map(file => file.relativePath)).toEqual(["Visible.md"]) + }) + it("captures lastModified timestamp", async () => { let root = new MockDirectoryHandle("root") let timestamp = 1234567890000 - root.addFile("Test.md", createMockFile("# Test", timestamp)) + root.addFile("Test.md", createMockFile("# Test"), timestamp) let files = await scanBackupFolder(root) diff --git a/src/lib/backup-sync.ts b/src/lib/backup-sync.ts index 63c65bf..6f1d8ed 100644 --- a/src/lib/backup-sync.ts +++ b/src/lib/backup-sync.ts @@ -84,14 +84,28 @@ let backupManifestSchema = z.object({ function computeDocLocations(docs: BackupDoc[]): Map { let docLocations = new Map() - let usedNames = new Map>() // parentPath -> used names (lowercase) + let usedNames = new Map>() + let sortedDocs = [...docs].sort((left, right) => { + let leftParentPath = left.path ?? "" + let rightParentPath = right.path ?? "" + if (leftParentPath !== rightParentPath) { + return leftParentPath.localeCompare(rightParentPath) + } - for (let doc of docs) { + let leftName = sanitizeFilename(left.title) + let rightName = sanitizeFilename(right.title) + if (leftName !== rightName) { + return leftName.localeCompare(rightName) + } + + return left.id.localeCompare(right.id) + }) + + for (let doc of sortedDocs) { let baseName = sanitizeFilename(doc.title) let hasAssets = doc.assets.length > 0 let parentPath = doc.path ?? "" - // Track used names at parent level for conflict detection if (!usedNames.has(parentPath)) usedNames.set(parentPath, new Set()) let used = usedNames.get(parentPath)! @@ -113,10 +127,17 @@ function computeDocLocations(docs: BackupDoc[]): Map { hasOwnFolder = false } - // Build asset filename map for this doc let assetFiles = new Map() let usedAssetNames = new Set() - for (let asset of doc.assets) { + let sortedAssets = [...doc.assets].sort((left, right) => { + let leftName = sanitizeFilename(left.name) || "image" + let rightName = sanitizeFilename(right.name) || "image" + if (leftName !== rightName) { + return leftName.localeCompare(rightName) + } + return left.id.localeCompare(right.id) + }) + for (let asset of sortedAssets) { let ext = getExtensionFromBlob(asset.blob) let assetBaseName = sanitizeFilename(asset.name) || "image" let fileName = assetBaseName + ext @@ -159,17 +180,14 @@ function transformContentForImport( content: string, assetFiles: Map, ): string { - // Transform local asset paths back to asset: references return content.replace( /!\[([^\]]*)\]\(assets\/([^)]+)\)/g, (match, alt, assetFilename) => { - // Find asset ID by filename for (let [id, filename] of assetFiles) { if (filename === assetFilename) { return `![${alt}](asset:${id})` } } - // If not found, keep the original local path (might be a manual addition) return match }, ) @@ -185,7 +203,6 @@ function computeExpectedStructure( for (let doc of docs) { let loc = docLocations.get(doc.id)! - // Add the directory path and all parent paths if (loc.dirPath) { let parts = loc.dirPath.split("/") for (let i = 1; i <= parts.length; i++) { @@ -193,13 +210,11 @@ function computeExpectedStructure( } } - // Add expected file if (!expectedFiles.has(loc.dirPath)) { expectedFiles.set(loc.dirPath, new Set()) } expectedFiles.get(loc.dirPath)!.add(loc.filename) - // If doc has assets, expect assets subfolder if (loc.hasOwnFolder && doc.assets.length > 0) { let assetsPath = loc.dirPath ? `${loc.dirPath}/assets` : "assets" expectedPaths.add(assetsPath) @@ -214,20 +229,31 @@ async function scanBackupFolder( ): Promise { let files: ScannedFile[] = [] + function getReferencedAssetNames(content: string): Set { + let references = new Set() + for (let match of content.matchAll(/!\[[^\]]*\]\(assets\/([^)]+)\)/g)) { + let filename = match[1] + if (!filename) continue + references.add(filename) + } + return references + } + async function scanDir( dir: FileSystemDirectoryHandle, relativePath: string, ): Promise { for await (let [name, handle] of dir.entries()) { let entryPath = relativePath ? `${relativePath}/${name}` : name + let loweredName = name.toLowerCase() if (handle.kind === "directory") { - // Skip dot directories and special directories if (name.startsWith(".")) continue + if (loweredName === "assets") continue let subDir = await dir.getDirectoryHandle(name) await scanDir(subDir, entryPath) - } else if (handle.kind === "file" && name.endsWith(".md")) { - // Skip manifest file + } else if (handle.kind === "file" && loweredName.endsWith(".md")) { + if (name.startsWith(".")) continue if (name === ".alkalye-manifest.json") continue let fileHandle = await dir.getFileHandle(name) @@ -235,24 +261,29 @@ async function scanBackupFolder( let content = await file.text() let lastModified = file.lastModified - // Check for assets folder let assets: { name: string; blob: Blob }[] = [] - try { - let assetsDir = await dir.getDirectoryHandle("assets") - for await (let [assetName, assetHandle] of assetsDir.entries()) { - if (assetHandle.kind === "file" && !assetName.startsWith(".")) { - let assetFileHandle = await assetsDir.getFileHandle(assetName) - let assetFile = await assetFileHandle.getFile() - assets.push({ name: assetName, blob: assetFile }) + let referencedAssets = getReferencedAssetNames(content) + if (referencedAssets.size > 0) { + let assetsDir = await dir + .getDirectoryHandle("assets") + .catch(() => null) + if (assetsDir) { + for (let assetName of referencedAssets) { + if (assetName.startsWith(".")) continue + try { + let assetFileHandle = await assetsDir.getFileHandle(assetName) + let assetFile = await assetFileHandle.getFile() + assets.push({ name: assetName, blob: assetFile }) + } catch { + continue + } } } - } catch { - // No assets folder } files.push({ relativePath: entryPath, - name: name.replace(/\.md$/, ""), + name: name.replace(/\.md$/i, ""), content, assets, lastModified, @@ -262,6 +293,9 @@ async function scanBackupFolder( } await scanDir(handle, "") + files.sort((left, right) => + left.relativePath.localeCompare(right.relativePath), + ) return files } diff --git a/src/lib/backup-test-helpers.ts b/src/lib/backup-test-helpers.ts index 87f33be..ff15da4 100644 --- a/src/lib/backup-test-helpers.ts +++ b/src/lib/backup-test-helpers.ts @@ -15,12 +15,56 @@ interface StoredFile { type: string } -function createMockFile(content: string, lastModified = Date.now()): File { - return new File([content], "test.md", { lastModified }) +function createMockFile(content: string): string { + return content +} + +class MockBlob implements Blob { + size: number + type: string + private data: Uint8Array + + constructor(content: string | Uint8Array, type: string) { + let raw = typeof content === "string" ? encodeText(content) : content + this.data = toStrictBytes(raw) + this.size = this.data.byteLength + this.type = type + } + + bytes(): Promise> { + return Promise.resolve(toStrictBytes(this.data)) + } + + arrayBuffer(): Promise { + return Promise.resolve(bytesToArrayBuffer(this.data)) + } + + text(): Promise { + return Promise.resolve(decodeBytes(this.data)) + } + + stream(): ReadableStream> { + let bytes = toStrictBytes(this.data) + return new ReadableStream>({ + start(controller) { + controller.enqueue(bytes) + controller.close() + }, + }) + } + + slice(start?: number, end?: number, contentType?: string): Blob { + let next = this.data.slice(start ?? 0, end ?? this.data.byteLength) + return new MockBlob(next, contentType ?? this.type) + } + + get [Symbol.toStringTag](): string { + return "Blob" + } } function createMockBlob(content: string, type = "image/png"): Blob { - return new Blob([content], { type }) + return new MockBlob(content, type) } async function readFileAtPath( @@ -82,6 +126,40 @@ function basename(relativePath: string): string { return parts[parts.length - 1] } +function encodeText(value: string): Uint8Array { + return new TextEncoder().encode(value) +} + +function bytesToArrayBuffer(bytes: Uint8Array): ArrayBuffer { + let buffer = new ArrayBuffer(bytes.byteLength) + new Uint8Array(buffer).set(bytes) + return buffer +} + +function toStrictBytes(bytes: Uint8Array): Uint8Array { + let buffer = bytesToArrayBuffer(bytes) + let strict: Uint8Array = new Uint8Array(buffer) + return strict +} + +function decodeBytes(bytes: Uint8Array): string { + return new TextDecoder().decode(bytes) +} + +async function readBlobBytes(blob: Blob): Promise> { + if (typeof blob.arrayBuffer === "function") { + let buffer = await blob.arrayBuffer() + return new Uint8Array(buffer) + } + + if (typeof blob.text === "function") { + let text = await blob.text() + return encodeText(text) + } + + throw new Error("Blob cannot be read in this environment") +} + class MockWritableFileStream implements FileSystemWritableFileStream { private stream = new WritableStream() private saveContent: (content: string) => void @@ -204,13 +282,30 @@ class MockFileHandle implements FileSystemFileHandle { if (!file) { return new File([""], this.name) } + + let bytes = + file.content !== null ? encodeText(file.content) : new Uint8Array(0) if (file.content === null && file.source) { - return file.source + bytes = await readBlobBytes(file.source) } - return new File([file.content ?? ""], this.name, { + + let compatibleFile = new File([bytesToArrayBuffer(bytes)], this.name, { lastModified: file.lastModified, type: file.type, }) + + if (typeof compatibleFile.text !== "function") { + Object.defineProperty(compatibleFile, "text", { + value: async () => decodeBytes(bytes), + }) + } + if (typeof compatibleFile.arrayBuffer !== "function") { + Object.defineProperty(compatibleFile, "arrayBuffer", { + value: async () => bytesToArrayBuffer(bytes), + }) + } + + return compatibleFile } async createWritable(): Promise { diff --git a/src/lib/backup.test.ts b/src/lib/backup.test.ts index 661e4f7..de8afd0 100644 --- a/src/lib/backup.test.ts +++ b/src/lib/backup.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest" -import { hashContent, syncFromBackup } from "./backup" +import { hashContent, syncFromBackup } from "./backup-engine" describe("hashContent", () => { it("returns consistent hash for same content", async () => { diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx deleted file mode 100644 index cdad7c9..0000000 --- a/src/lib/backup.tsx +++ /dev/null @@ -1,1901 +0,0 @@ -import { useState, useEffect, useRef } from "react" -import { create } from "zustand" -import { persist } from "zustand/middleware" -import { useAccount, useCoState } from "jazz-tools/react" -import { co, type ResolveQuery, Group, FileStream } from "jazz-tools" -import { createImage } from "jazz-tools/media" -import { FolderOpen, AlertCircle } from "lucide-react" -import { - UserAccount, - Document, - Space, - Asset, - ImageAsset, - VideoAsset, -} from "@/schema" -import { getDocumentTitle } from "@/lib/document-utils" -import { getPath, parseFrontmatter } from "@/editor/frontmatter" -import { Button } from "@/components/ui/button" -import { get as idbGet, set as idbSet, del as idbDel } from "idb-keyval" -import { - computeDocLocations, - transformContentForBackup, - computeExpectedStructure, - scanBackupFolder, - readManifest, - writeManifest, - transformContentForImport, - type BackupDoc, - type ManifestEntry, - type ScannedFile, -} from "@/lib/backup-sync" - -export { - BackupSubscriber, - SpacesBackupSubscriber, - BackupSettings, - SpaceBackupSettings, - useSpaceBackupPath, - getSpaceBackupPath, - setSpaceBackupPath, - clearSpaceBackupPath, - enableBackup, - disableBackup, - changeBackupDirectory, - checkBackupPermission, - // Exported for testing - hashContent, - syncBackup, - syncFromBackup, - type ScannedFile, -} - -// File System Access API type augmentation -declare global { - interface FileSystemObserver { - observe( - handle: FileSystemDirectoryHandle, - options?: { recursive?: boolean }, - ): Promise - disconnect(): void - } - - interface Window { - FileSystemObserver?: { - new ( - onChange: (records: unknown[], observer: FileSystemObserver) => void, - ): FileSystemObserver - } - showDirectoryPicker(options?: { - mode?: "read" | "readwrite" - }): Promise - } - interface FileSystemDirectoryHandle { - entries(): AsyncIterableIterator<[string, FileSystemHandle]> - queryPermission(options: { - mode: "read" | "readwrite" - }): Promise<"granted" | "denied" | "prompt"> - requestPermission(options: { - mode: "read" | "readwrite" - }): Promise<"granted" | "denied" | "prompt"> - } -} - -let BACKUP_DEBOUNCE_MS = 1200 - -let HANDLE_STORAGE_KEY = "backup-directory-handle" -let preferredRelativePathByDocId = new Map() -let recentImportedRelativePaths = new Map() -let RECENT_IMPORT_WINDOW_MS = 30_000 -let spaceLastPullAtById = new Map() - -interface BackupState { - enabled: boolean - bidirectional: boolean - directoryName: string | null - lastBackupAt: string | null - lastPullAt: string | null - lastError: string | null - setEnabled: (enabled: boolean) => void - setBidirectional: (bidirectional: boolean) => void - setDirectoryName: (name: string | null) => void - setLastBackupAt: (date: string | null) => void - setLastPullAt: (date: string | null) => void - setLastError: (error: string | null) => void - reset: () => void -} - -let useBackupStore = create()( - persist( - set => ({ - enabled: false, - bidirectional: true, - directoryName: null, - lastBackupAt: null, - lastPullAt: null, - lastError: null, - setEnabled: enabled => set({ enabled }), - setBidirectional: bidirectional => set({ bidirectional }), - setDirectoryName: directoryName => set({ directoryName }), - setLastBackupAt: lastBackupAt => set({ lastBackupAt }), - setLastPullAt: lastPullAt => set({ lastPullAt }), - setLastError: lastError => set({ lastError }), - reset: () => - set({ - enabled: false, - bidirectional: true, - directoryName: null, - lastBackupAt: null, - lastPullAt: null, - lastError: null, - }), - }), - { name: "backup-settings" }, - ), -) - -type LoadedDocument = co.loaded< - typeof Document, - { content: true; assets: { $each: { image: true; video: true } } } -> - -type DocumentList = co.loaded>> -type Account = co.loaded - -interface SyncFromBackupResult { - created: number - updated: number - deleted: number - errors: string[] -} - -interface ScannedAssetHash { - name: string - hash: string -} - -let backupQuery = { - root: { - documents: { - $each: { content: true, assets: { $each: { image: true, video: true } } }, - $onError: "catch", - }, - }, -} as const satisfies ResolveQuery - -function BackupSubscriber() { - let { - enabled, - bidirectional, - lastPullAt, - setLastBackupAt, - setLastPullAt, - setLastError, - setEnabled, - setDirectoryName, - } = useBackupStore() - let me = useAccount(UserAccount, { resolve: backupQuery }) - let debounceRef = useRef | null>(null) - let lastContentHashRef = useRef("") - let isPushingRef = useRef(false) - let isPullingRef = useRef(false) - - // Push to filesystem (backup) - useEffect(() => { - if (!enabled || !me.$isLoaded) return - - let docs = me.root?.documents - if (!docs?.$isLoaded) return - - // Compute content hash to detect changes - let activeDocs = [...docs].filter(d => d?.$isLoaded && !d.deletedAt) - let contentHash = activeDocs - .map(d => `${d.$jazz.id}:${d.updatedAt?.getTime()}`) - .sort() - .join("|") - - if (contentHash === lastContentHashRef.current) return - lastContentHashRef.current = contentHash - - // Debounce backup - if (debounceRef.current) clearTimeout(debounceRef.current) - debounceRef.current = setTimeout(async () => { - try { - let handle = await getBackupHandle() - if (!handle) { - setEnabled(false) - setDirectoryName(null) - setLastError("Permission lost - please re-enable backup") - return - } - - let loadedDocs = activeDocs.filter( - (d): d is LoadedDocument => d?.$isLoaded === true, - ) - let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) - isPushingRef.current = true - await syncBackup(handle, backupDocs) - - setLastBackupAt(new Date().toISOString()) - setLastError(null) - } catch (e) { - setLastError(e instanceof Error ? e.message : "Backup failed") - } finally { - isPushingRef.current = false - } - }, BACKUP_DEBOUNCE_MS) - - return () => { - if (debounceRef.current) clearTimeout(debounceRef.current) - } - }, [enabled, me, setLastBackupAt, setLastError, setEnabled, setDirectoryName]) - - // Pull from filesystem (import changes) - only supported with FileSystemObserver - useEffect(() => { - if (!enabled || !bidirectional || !me.$isLoaded) return - if (!supportsFileSystemWatch()) return - - let docs = me.root?.documents - if (!docs?.$isLoaded) return - - async function doPull() { - try { - if (isPushingRef.current) return - if (isPullingRef.current) return - isPullingRef.current = true - let handle = await getBackupHandle() - if (!handle) return - - if (!docs.$isLoaded || !isDocumentList(docs)) return - let result = await syncFromBackup( - handle, - docs, - true, - toTimestamp(lastPullAt), - ) - if (result.errors.length > 0) { - console.warn("Backup pull errors:", result.errors) - } - - setLastPullAt(new Date().toISOString()) - } catch (e) { - console.error("Backup pull failed:", e) - } finally { - isPullingRef.current = false - } - } - - // Set up observer for real-time file change detection - let watchAborted = false - let stopWatching: (() => void) | null = null - - async function setupWatch() { - let handle = await getBackupHandle() - if (!handle) return - - let stop = await observeDirectoryChanges(handle, () => { - if (!watchAborted) doPull() - }) - if (watchAborted) { - stop?.() - return - } - stopWatching = stop - } - - setupWatch() - - return () => { - watchAborted = true - stopWatching?.() - } - }, [enabled, bidirectional, me, setLastPullAt, lastPullAt]) - - return null -} - -// Export for settings UI -async function enableBackup(): Promise<{ - success: boolean - directoryName?: string - error?: string -}> { - let handle = await requestBackupDirectory() - if (!handle) return { success: false, error: "Cancelled" } - - useBackupStore.getState().setEnabled(true) - useBackupStore.getState().setDirectoryName(handle.name) - useBackupStore.getState().setLastError(null) - - return { success: true, directoryName: handle.name } -} - -async function disableBackup(): Promise { - await clearHandle() - useBackupStore.getState().reset() -} - -async function changeBackupDirectory(): Promise<{ - success: boolean - directoryName?: string - error?: string -}> { - let handle = await requestBackupDirectory() - if (!handle) return { success: false, error: "Cancelled" } - - useBackupStore.getState().setDirectoryName(handle.name) - useBackupStore.getState().setLastError(null) - // Trigger immediate backup by clearing the hash - return { success: true, directoryName: handle.name } -} - -async function checkBackupPermission(): Promise { - let handle = await getBackupHandle() - return handle !== null -} - -// Settings UI component - -function BackupSettings() { - let { - enabled, - bidirectional, - directoryName, - lastBackupAt, - lastPullAt, - lastError, - setBidirectional, - } = useBackupStore() - let [isLoading, setIsLoading] = useState(false) - - if (!isBackupSupported()) { - return ( -
-

- Local Backup -

-
-
- -
-

- Local backup requires a Chromium-based browser (Chrome, Edge, - Brave, or Opera). -

-

- Safari and Firefox do not support the File System Access API - needed for this feature. -

-
-
-
-
- ) - } - - async function handleEnable() { - setIsLoading(true) - await enableBackup() - setIsLoading(false) - } - - async function handleDisable() { - setIsLoading(true) - await disableBackup() - setIsLoading(false) - } - - async function handleChangeDirectory() { - setIsLoading(true) - await changeBackupDirectory() - setIsLoading(false) - } - - let lastBackupDate = lastBackupAt ? new Date(lastBackupAt) : null - let formattedLastBackup = lastBackupDate - ? lastBackupDate.toLocaleString() - : null - - let lastPullDate = lastPullAt ? new Date(lastPullAt) : null - let formattedLastPull = lastPullDate ? lastPullDate.toLocaleString() : null - - return ( -
-

- Local Backup -

-
- {enabled ? ( - <> -
- - - {bidirectional ? "Syncing" : "Backing up"} to folder - -
-

- Folder: {directoryName} -

- {formattedLastBackup && ( -

- Last backup: {formattedLastBackup} -

- )} - {bidirectional && formattedLastPull && ( -

- Last sync: {formattedLastPull} -

- )} - {lastError && ( -
- - {lastError} -
- )} -
- -

- {supportsFileSystemWatch() - ? "When enabled, changes made in the backup folder will be imported into Alkalye." - : "Requires a Chromium-based browser with File System Observer support."} -

-
-
- - -
- - ) : ( - <> -
- Automatic backup disabled -
-

- Automatically back up your documents to a folder on this device. -

- - - )} -
-
- ) -} - -// Space-specific backup path settings stored in localStorage - -let SPACE_BACKUP_KEY_PREFIX = "backup-settings-space-" - -interface SpaceBackupState { - directoryName: string | null -} - -function getSpaceBackupPath(spaceId: string): string | null { - try { - let key = getSpaceBackupStorageKey(spaceId) - let stored = localStorage.getItem(key) - if (!stored) return null - let parsed = JSON.parse(stored) - if (!isSpaceBackupState(parsed)) return null - return parsed.directoryName - } catch { - return null - } -} - -function setSpaceBackupPath(spaceId: string, directoryName: string): void { - let key = getSpaceBackupStorageKey(spaceId) - let state: SpaceBackupState = { directoryName } - localStorage.setItem(key, JSON.stringify(state)) -} - -function clearSpaceBackupPath(spaceId: string): void { - let key = getSpaceBackupStorageKey(spaceId) - localStorage.removeItem(key) -} - -function useSpaceBackupPath(spaceId: string): { - directoryName: string | null - setDirectoryName: (name: string | null) => void -} { - let [directoryName, setDirectoryNameState] = useState(() => - getSpaceBackupPath(spaceId), - ) - - function setDirectoryName(name: string | null) { - if (name) { - setSpaceBackupPath(spaceId, name) - } else { - clearSpaceBackupPath(spaceId) - } - setDirectoryNameState(name) - } - - return { directoryName, setDirectoryName } -} - -interface SpaceBackupSettingsProps { - spaceId: string - isAdmin: boolean -} - -function SpaceBackupSettings({ spaceId, isAdmin }: SpaceBackupSettingsProps) { - let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) - let [isLoading, setIsLoading] = useState(false) - - if (!isBackupSupported()) { - return ( -
-

- Local Backup -

-
-
- -
-

- Local backup requires a Chromium-based browser (Chrome, Edge, - Brave, or Opera). -

-

- Safari and Firefox do not support the File System Access API - needed for this feature. -

-
-
-
-
- ) - } - - async function handleChooseFolder() { - setIsLoading(true) - try { - let handle = await window.showDirectoryPicker({ mode: "readwrite" }) - // Store handle in IndexedDB with space-specific key - await idbSet(`${HANDLE_STORAGE_KEY}-space-${spaceId}`, handle) - setDirectoryName(handle.name) - } catch (e) { - if (!(e instanceof Error && e.name === "AbortError")) { - console.error("Failed to select folder:", e) - } - } finally { - setIsLoading(false) - } - } - - async function handleChangeFolder() { - await handleChooseFolder() - } - - async function handleClear() { - setIsLoading(true) - try { - await idbDel(`${HANDLE_STORAGE_KEY}-space-${spaceId}`) - setDirectoryName(null) - } finally { - setIsLoading(false) - } - } - - return ( -
-

- Local Backup -

-
- {directoryName ? ( - <> -
- - Backup folder set -
-

- Folder: {directoryName} -

-
- - -
- - ) : ( - <> -
- No backup folder set -
-

- Set a backup folder for this space's documents. -

- - - )} -
-
- ) -} - -// Component that subscribes to all spaces and renders backup subscribers for each - -function SpacesBackupSubscriber() { - let me = useAccount(UserAccount, { resolve: spacesBackupQuery }) - let [storageVersion, setStorageVersion] = useState(0) - - // Re-render on storage changes (for when user sets backup path in settings) - useEffect(() => { - function handleStorageChange(e: StorageEvent) { - if (e.key?.startsWith(SPACE_BACKUP_KEY_PREFIX)) { - setStorageVersion(v => v + 1) - } - } - - window.addEventListener("storage", handleStorageChange) - return () => window.removeEventListener("storage", handleStorageChange) - }, []) - - // Compute spaces with backup paths - recomputed when me changes or storage changes - let spacesWithBackup = getSpacesWithBackup(me, storageVersion) - - return ( - <> - {spacesWithBackup.map(spaceId => ( - - ))} - - ) -} - -// Exported for testing -async function hashContent(content: string): Promise { - // Simple hash using built-in crypto - let encoder = new TextEncoder() - let data = encoder.encode(content) - let hashBuffer = await crypto.subtle.digest("SHA-256", data) - let hashArray = Array.from(new Uint8Array(hashBuffer)) - return hashArray - .map(b => b.toString(16).padStart(2, "0")) - .join("") - .slice(0, 16) -} - -async function syncBackup( - handle: FileSystemDirectoryHandle, - docs: BackupDoc[], -): Promise { - await performSyncBackup(handle, docs) -} - -async function syncFromBackup( - handle: FileSystemDirectoryHandle, - targetDocs: DocumentList, - canWrite: boolean, - lastPullAtMs: number | null = null, -): Promise { - let result: SyncFromBackupResult = { - created: 0, - updated: 0, - deleted: 0, - errors: [], - } - - let manifest = await readManifest(handle) - let scannedFiles = await scanBackupFolder(handle) - let listOwner = targetDocs.$jazz.owner - - // Build maps for lookup - let manifestByPath = new Map( - manifest?.entries.map(e => [e.relativePath, e]) ?? [], - ) - let scannedByPath = new Map(scannedFiles.map(f => [f.relativePath, f])) - let matchedManifestDocIds = new Set() - - // Process new and updated files - for (let file of scannedFiles) { - try { - let contentHash = await hashContent(file.content) - let scannedAssetHashes = await hashScannedAssets(file.assets) - let manifestEntry = manifestByPath.get(file.relativePath) - if (manifestEntry) { - matchedManifestDocIds.add(manifestEntry.docId) - if (lastPullAtMs !== null && file.lastModified <= lastPullAtMs) { - continue - } - } - - if (!manifestEntry) { - let movedEntry = findMovedManifestEntry( - manifest, - scannedByPath, - matchedManifestDocIds, - file, - contentHash, - ) - if (movedEntry) { - manifestEntry = movedEntry - matchedManifestDocIds.add(movedEntry.docId) - } - } - - if (!manifestEntry) { - // New file - create document - if (!canWrite) { - result.errors.push(`Cannot create ${file.name}: no write permission`) - continue - } - if (wasRecentlyImported(file.relativePath)) continue - let newDocId = await createDocFromFile(file, targetDocs, listOwner) - preferredRelativePathByDocId.set(newDocId, file.relativePath) - markRecentlyImported(file.relativePath) - result.created++ - } else if ( - manifestEntry.contentHash !== contentHash || - manifestEntry.relativePath !== file.relativePath || - !areScannedAssetsInSync(manifestEntry.assets, scannedAssetHashes) - ) { - preferredRelativePathByDocId.set(manifestEntry.docId, file.relativePath) - // File changed or moved - update document - if (!canWrite) { - result.errors.push(`Cannot update ${file.name}: no write permission`) - continue - } - let didUpdate = await updateDocFromFile( - file, - manifestEntry.docId, - manifestEntry, - targetDocs, - ) - if (didUpdate) { - result.updated++ - } else { - result.errors.push( - `Skipped update for ${file.relativePath}: target document not loaded`, - ) - } - } - } catch (err) { - result.errors.push( - `Failed to process ${file.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, - ) - } - } - - // Handle deletions (files in manifest but not on disk) - if (manifest && canWrite) { - for (let entry of manifest.entries) { - if (matchedManifestDocIds.has(entry.docId)) continue - if (!scannedByPath.has(entry.relativePath)) { - try { - let doc = targetDocs.find(d => d?.$jazz.id === entry.docId) - if (doc?.$isLoaded && !doc.deletedAt) { - // Soft delete - doc.$jazz.set("deletedAt", new Date()) - doc.$jazz.set("updatedAt", new Date()) - result.deleted++ - } - } catch (err) { - result.errors.push( - `Failed to delete ${entry.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, - ) - } - } - } - } - - return result -} - -// ============================================================================= -// Helper functions (used by exported functions above) -// ============================================================================= - -// Space backup subscriber - handles backup sync for a single space - -let SPACE_BACKUP_DEBOUNCE_MS = 1200 - -interface SpaceBackupSubscriberProps { - spaceId: string -} - -function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { - let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) - let debounceRef = useRef | null>(null) - let lastContentHashRef = useRef("") - let isPushingRef = useRef(false) - let isPullingRef = useRef(false) - - // Load space with documents - let space = useCoState(Space, spaceId, { - resolve: { - documents: { - $each: { content: true, assets: { $each: { image: true } } }, - $onError: "catch", - }, - }, - }) - - // Push to filesystem (backup) - useEffect(() => { - // Skip if no backup folder configured - if (!directoryName) return - // Skip if space not loaded - if (!space?.$isLoaded || !space.documents?.$isLoaded) return - - let docs = space.documents - let activeDocs = [...docs].filter(d => d?.$isLoaded && !d.deletedAt) - - // Compute content hash to detect changes - let contentHash = activeDocs - .map(d => `${d.$jazz.id}:${d.updatedAt?.getTime()}`) - .sort() - .join("|") - - if (contentHash === lastContentHashRef.current) return - lastContentHashRef.current = contentHash - - // Debounce backup - if (debounceRef.current) clearTimeout(debounceRef.current) - debounceRef.current = setTimeout(async () => { - try { - let handle = await getSpaceBackupHandle(spaceId) - if (!handle) { - // Permission lost, clear the setting - setDirectoryName(null) - return - } - - let loadedDocs = activeDocs.filter( - (d): d is LoadedDocument => d?.$isLoaded === true, - ) - let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) - isPushingRef.current = true - await syncBackup(handle, backupDocs) - } catch (e) { - console.error(`Space backup failed for ${spaceId}:`, e) - } finally { - isPushingRef.current = false - } - }, SPACE_BACKUP_DEBOUNCE_MS) - - return () => { - if (debounceRef.current) clearTimeout(debounceRef.current) - } - }, [directoryName, space, spaceId, setDirectoryName]) - - // Pull from filesystem (import changes) - only supported with FileSystemObserver - useEffect(() => { - // Skip if no backup folder configured - if (!directoryName) return - // Skip if space not loaded - if (!space?.$isLoaded || !space.documents?.$isLoaded) return - if (!supportsFileSystemWatch()) return - - let docs = space.documents - - // Check permissions for writing - let spaceGroup = - space.$jazz.owner instanceof Group ? space.$jazz.owner : null - let canWrite = - spaceGroup?.myRole() === "admin" || spaceGroup?.myRole() === "writer" - - async function doPull() { - try { - if (isPushingRef.current) return - if (isPullingRef.current) return - isPullingRef.current = true - let handle = await getSpaceBackupHandle(spaceId) - if (!handle) return - - if (!docs.$isLoaded || !isDocumentList(docs)) return - let result = await syncFromBackup( - handle, - docs, - canWrite, - spaceLastPullAtById.get(spaceId) ?? null, - ) - if (result.errors.length > 0) { - console.warn(`Space ${spaceId} pull errors:`, result.errors) - } - spaceLastPullAtById.set(spaceId, Date.now()) - } catch (e) { - console.error(`Space backup pull failed for ${spaceId}:`, e) - } finally { - isPullingRef.current = false - } - } - - // Set up observer for real-time file change detection - let watchAborted = false - let stopWatching: (() => void) | null = null - - async function setupWatch() { - let handle = await getSpaceBackupHandle(spaceId) - if (!handle) return - - let stop = await observeDirectoryChanges(handle, () => { - if (!watchAborted) doPull() - }) - if (watchAborted) { - stop?.() - return - } - stopWatching = stop - } - - setupWatch() - - return () => { - watchAborted = true - stopWatching?.() - } - }, [directoryName, space, spaceId]) - - return null -} - -let spacesBackupQuery = { - root: { - spaces: true, - }, -} as const satisfies ResolveQuery - -async function getStoredHandle(): Promise { - try { - let handle = await idbGet(HANDLE_STORAGE_KEY) - return handle ?? null - } catch { - return null - } -} - -async function storeHandle(handle: FileSystemDirectoryHandle): Promise { - await idbSet(HANDLE_STORAGE_KEY, handle) -} - -async function clearHandle(): Promise { - await idbDel(HANDLE_STORAGE_KEY) -} - -async function verifyPermission( - handle: FileSystemDirectoryHandle, -): Promise { - let opts: { mode: "readwrite" } = { mode: "readwrite" } - if ((await handle.queryPermission(opts)) === "granted") return true - if ((await handle.requestPermission(opts)) === "granted") return true - return false -} - -async function requestBackupDirectory(): Promise { - try { - let handle = await window.showDirectoryPicker({ mode: "readwrite" }) - await storeHandle(handle) - return handle - } catch (e) { - if (e instanceof Error && e.name === "AbortError") return null - throw e - } -} - -async function getBackupHandle(): Promise { - let handle = await getStoredHandle() - if (!handle) return null - let hasPermission = await verifyPermission(handle) - if (!hasPermission) return null - return handle -} - -async function prepareBackupDoc(doc: LoadedDocument): Promise { - let content = doc.content?.toString() ?? "" - let title = getDocumentTitle(doc) - let path = getPath(content) - let updatedAtMs = doc.updatedAt?.getTime() ?? 0 - - let assets: BackupDoc["assets"] = [] - if (doc.assets?.$isLoaded) { - for (let asset of [...doc.assets]) { - if (!asset?.$isLoaded) continue - - let blob: Blob | undefined - if (asset.type === "image" && asset.image?.$isLoaded) { - let original = asset.image.original - if (original?.$isLoaded) { - blob = original.toBlob() - } - } else if (asset.type === "video" && asset.video?.$isLoaded) { - blob = await asset.video.toBlob() - } - - if (blob) { - assets.push({ id: asset.$jazz.id, name: asset.name, blob }) - } - } - } - - return { id: doc.$jazz.id, title, content, path, updatedAtMs, assets } -} - -async function getOrCreateDirectory( - parent: FileSystemDirectoryHandle, - path: string, -): Promise { - let parts = path.split("/").filter(Boolean) - let current = parent - for (let part of parts) { - current = await current.getDirectoryHandle(part, { create: true }) - } - return current -} - -async function writeFile( - dir: FileSystemDirectoryHandle, - name: string, - content: string | Blob, -): Promise { - let file = await dir.getFileHandle(name, { create: true }) - let writable = await file.createWritable() - await writable.write(content) - await writable.close() -} - -async function deleteFile( - dir: FileSystemDirectoryHandle, - name: string, -): Promise { - try { - await dir.removeEntry(name) - } catch { - // File doesn't exist, ignore - } -} - -async function listFiles(dir: FileSystemDirectoryHandle): Promise { - let files: string[] = [] - for await (let [name, handle] of dir.entries()) { - if (handle.kind === "file") files.push(name) - } - return files -} - -async function listDirectories( - dir: FileSystemDirectoryHandle, -): Promise { - let dirs: string[] = [] - for await (let [name, handle] of dir.entries()) { - if (handle.kind === "directory") dirs.push(name) - } - return dirs -} - -async function performSyncBackup( - handle: FileSystemDirectoryHandle, - docs: BackupDoc[], -): Promise { - let docLocations = computeDocLocations(docs) - let existingManifest = await readManifest(handle) - let existingEntriesByDocId = new Map( - existingManifest?.entries.map(entry => [entry.docId, entry]) ?? [], - ) - let manifestEntries: { - docId: string - relativePath: string - locationKey: string - contentHash: string - lastSyncedAt: string - assets: { id: string; name: string; hash: string }[] - }[] = [] - let hasFilesystemChanges = false - let nowIso = new Date().toISOString() - - // Write only changed documents and assets - for (let doc of docs) { - let loc = docLocations.get(doc.id)! - let locationKey = getDocLocationKey(doc) - let computedRelativePath = loc.dirPath - ? `${loc.dirPath}/${loc.filename}` - : loc.filename - let existingEntry = existingEntriesByDocId.get(doc.id) - let preferredRelativePath = preferredRelativePathByDocId.get(doc.id) - let finalRelativePath = computedRelativePath - if (existingEntry) { - if (existingEntry.locationKey === locationKey) { - finalRelativePath = existingEntry.relativePath - } - } - if (preferredRelativePath) { - finalRelativePath = preferredRelativePath - } - let finalLocation = buildLocationFromRelativePath(loc, finalRelativePath) - docLocations.set(doc.id, finalLocation) - loc = finalLocation - - let dir = loc.dirPath - ? await getOrCreateDirectory(handle, loc.dirPath) - : handle - - let exportedContent = transformContentForBackup(doc.content, loc.assetFiles) - let contentHash = await hashContent(exportedContent) - let relativePath = finalRelativePath - let assets: { id: string; name: string; hash: string }[] = [] - for (let asset of doc.assets) { - let filename = loc.assetFiles.get(asset.id)! - assets.push({ - id: asset.id, - name: filename, - hash: await hashBlob(asset.blob), - }) - } - - let shouldWriteDoc = - !existingEntry || - existingEntry.relativePath !== relativePath || - existingEntry.contentHash !== contentHash || - !areManifestAssetsEqual(existingEntry.assets, assets) - - if (shouldWriteDoc) { - hasFilesystemChanges = true - await writeFile(dir, loc.filename, exportedContent) - - // Write assets if any - if (doc.assets.length > 0) { - let assetsDir = await dir.getDirectoryHandle("assets", { create: true }) - for (let asset of doc.assets) { - let filename = loc.assetFiles.get(asset.id)! - await writeFile(assetsDir, filename, asset.blob) - } - } - } - - manifestEntries.push({ - docId: doc.id, - relativePath, - locationKey, - contentHash, - lastSyncedAt: shouldWriteDoc - ? nowIso - : (existingEntry?.lastSyncedAt ?? nowIso), - assets, - }) - } - - let docsChanged = - existingEntriesByDocId.size !== manifestEntries.length || - hasFilesystemChanges - - if (docsChanged) { - // Clean up orphaned files and directories - await cleanupOrphanedFiles(handle, docs, docLocations) - - await writeManifest(handle, { - version: 1, - entries: manifestEntries, - lastSyncAt: nowIso, - }) - - for (let entry of manifestEntries) { - recentImportedRelativePaths.delete(entry.relativePath) - } - } - - for (let doc of docs) { - preferredRelativePathByDocId.delete(doc.id) - } -} - -async function cleanupOrphanedFiles( - handle: FileSystemDirectoryHandle, - docs: BackupDoc[], - docLocations: Map< - string, - ReturnType extends Map - ? V - : never - >, -): Promise { - let { expectedPaths, expectedFiles } = computeExpectedStructure( - docs, - docLocations, - ) - let expectedAssetFilesByDir = new Map>() - for (let doc of docs) { - let location = docLocations.get(doc.id) - if (!location || !location.hasOwnFolder) continue - let assetsPath = location.dirPath ? `${location.dirPath}/assets` : "assets" - expectedAssetFilesByDir.set( - assetsPath, - new Set(location.assetFiles.values()), - ) - } - - async function cleanAssetsDirectory( - dir: FileSystemDirectoryHandle, - path: string, - ): Promise { - let hasContent = false - let expected = expectedAssetFilesByDir.get(path) ?? new Set() - - for await (let [name, child] of dir.entries()) { - if (name.startsWith(".")) continue - if (child.kind === "directory") { - await dir.removeEntry(name, { recursive: true }) - continue - } - if (expected.has(name)) { - hasContent = true - continue - } - await deleteFile(dir, name) - } - - return hasContent - } - - async function cleanDir( - dir: FileSystemDirectoryHandle, - path: string, - ): Promise { - let subdirs = await listDirectories(dir) - let hasContent = false - - for (let subdir of subdirs) { - let subPath = path ? `${path}/${subdir}` : subdir - - if (subdir === "assets" && expectedPaths.has(subPath)) { - let subHandle = await dir.getDirectoryHandle(subdir) - let assetsHasContent = await cleanAssetsDirectory(subHandle, subPath) - if (assetsHasContent) hasContent = true - continue - } - - if (expectedPaths.has(subPath)) { - let subHandle = await dir.getDirectoryHandle(subdir) - let subHasContent = await cleanDir(subHandle, subPath) - if (subHasContent) hasContent = true - } else { - // Directory not expected, remove it - try { - await dir.removeEntry(subdir, { recursive: true }) - } catch { - // Ignore errors - } - } - } - - // Clean files in this directory - let expected = expectedFiles.get(path) ?? new Set() - let files = await listFiles(dir) - for (let file of files) { - if (file.endsWith(".md")) { - if (expected.has(file)) { - hasContent = true - } else { - await deleteFile(dir, file) - } - } - } - - return hasContent - } - - await cleanDir(handle, "") -} - -async function createDocFromFile( - file: ScannedFile, - targetDocs: DocumentList, - listOwner: Group | Account, -): Promise { - // Create doc-specific group with list owner as parent - let docGroup = Group.create() - if (listOwner instanceof Group) { - docGroup.addMember(listOwner) - } - - let now = new Date() - - // Create assets - let docAssets: co.loaded[] = [] - let assetFilesById = new Map() - for (let assetFile of file.assets) { - let asset = await createAssetFromBlob(assetFile, docGroup, now) - docAssets.push(asset) - assetFilesById.set(asset.$jazz.id, assetFile.name) - } - - let transformedContent = transformContentForImport( - file.content, - assetFilesById, - ) - let content = applyPathFromRelativePath( - transformedContent, - file.relativePath, - file.assets.length > 0, - ) - - let newDoc = Document.create( - { - version: 1, - content: co.plainText().create(content, docGroup), - assets: - docAssets.length > 0 - ? co.list(Asset).create(docAssets, docGroup) - : undefined, - createdAt: now, - updatedAt: now, - }, - docGroup, - ) - - targetDocs.$jazz.push(newDoc) - return newDoc.$jazz.id -} - -async function updateDocFromFile( - file: ScannedFile, - docId: string, - manifestEntry: ManifestEntry, - targetDocs: DocumentList, -): Promise { - let doc = targetDocs.find( - (d): d is LoadedDocument => d?.$isLoaded === true && d.$jazz.id === docId, - ) - if (!doc || !doc.content?.$isLoaded) { - return false - } - - let assetFilesById = await syncDocAssetsFromFile(doc, file, manifestEntry) - let content = applyPathFromRelativePath( - transformContentForImport(file.content, assetFilesById), - file.relativePath, - file.assets.length > 0, - ) - - doc.content.$jazz.applyDiff(content) - doc.$jazz.set("updatedAt", new Date()) - return true -} - -async function syncDocAssetsFromFile( - doc: LoadedDocument, - file: ScannedFile, - manifestEntry: ManifestEntry, -): Promise> { - if (file.assets.length === 0) { - let manifestFilesById = getAssetFilesByIdFromManifest(manifestEntry) - let keepsManifestRefs = Array.from(manifestFilesById.values()).some( - filename => file.content.includes(`assets/${filename}`), - ) - if (keepsManifestRefs) { - return manifestFilesById - } - if (doc.assets?.$isLoaded) { - for (let i = doc.assets.length - 1; i >= 0; i--) { - doc.assets.$jazz.splice(i, 1) - } - } - return new Map() - } - - if (!doc.assets) { - doc.$jazz.set("assets", co.list(Asset).create([], doc.$jazz.owner)) - } - if (!doc.assets?.$isLoaded) { - return getAssetFilesByIdFromManifest(manifestEntry) - } - - let currentAssets = Array.from(doc.assets).filter( - (asset): asset is co.loaded => asset?.$isLoaded === true, - ) - let assetsById = new Map(currentAssets.map(asset => [asset.$jazz.id, asset])) - let fileAssetsWithHash = await Promise.all( - file.assets.map(async asset => ({ - name: asset.name, - blob: asset.blob, - hash: await hashBlob(asset.blob), - })), - ) - - let manifestByName = new Map( - manifestEntry.assets.map(asset => [asset.name, asset]), - ) - let manifestByHash = new Map() - for (let asset of manifestEntry.assets) { - if (!manifestByHash.has(asset.hash)) { - manifestByHash.set(asset.hash, []) - } - manifestByHash.get(asset.hash)?.push(asset) - } - - let keepAssetIds = new Set() - let assetFilesById = new Map() - - for (let fileAsset of fileAssetsWithHash) { - let matchedByName = manifestByName.get(fileAsset.name) - let matchedId = - matchedByName?.id && assetsById.has(matchedByName.id) - ? matchedByName.id - : null - - if (!matchedId) { - let byHash = manifestByHash.get(fileAsset.hash) ?? [] - for (let candidate of byHash) { - if (!candidate.id || keepAssetIds.has(candidate.id)) continue - if (!assetsById.has(candidate.id)) continue - matchedId = candidate.id - break - } - } - - if (matchedId) { - let existing = assetsById.get(matchedId) - if (!existing) continue - - let shouldUpdateBinary = - matchedByName?.id === matchedId - ? matchedByName.hash !== fileAsset.hash - : false - - if (shouldUpdateBinary) { - matchedId = await syncExistingAssetFromFile( - doc, - existing, - matchedId, - fileAsset, - ) - } - - let updatedAsset = doc.assets.find( - asset => asset?.$isLoaded && asset.$jazz.id === matchedId, - ) - if (!updatedAsset) continue - assetsById.set(matchedId, updatedAsset) - if (updatedAsset.name !== removeExtension(fileAsset.name)) { - updatedAsset.$jazz.applyDiff({ name: removeExtension(fileAsset.name) }) - } - - keepAssetIds.add(matchedId) - assetFilesById.set(matchedId, fileAsset.name) - continue - } - - let created = await createAssetFromBlob( - fileAsset, - doc.$jazz.owner, - new Date(), - ) - doc.assets.$jazz.push(created) - keepAssetIds.add(created.$jazz.id) - assetFilesById.set(created.$jazz.id, fileAsset.name) - } - - for (let i = doc.assets.length - 1; i >= 0; i--) { - let asset = doc.assets[i] - if (!asset?.$isLoaded) continue - if (keepAssetIds.has(asset.$jazz.id)) continue - doc.assets.$jazz.splice(i, 1) - } - - if (fileAssetsWithHash.length === 0) { - return new Map() - } - - return assetFilesById -} - -async function syncExistingAssetFromFile( - doc: LoadedDocument, - existing: co.loaded, - assetId: string, - fileAsset: { name: string; blob: Blob }, -): Promise { - let nextType = fileAsset.blob.type.startsWith("video/") ? "video" : "image" - if (existing.type !== nextType) { - let index = - doc.assets?.findIndex(asset => asset?.$jazz.id === assetId) ?? -1 - if (index === -1 || !doc.assets) return assetId - doc.assets.$jazz.splice(index, 1) - let replacement = await createAssetFromBlob( - fileAsset, - doc.$jazz.owner, - existing.createdAt, - ) - doc.assets.$jazz.push(replacement) - return replacement.$jazz.id - } - - if (existing.type === "video") { - let stream = await FileStream.createFromBlob(fileAsset.blob, { - owner: doc.$jazz.owner, - }) - existing.$jazz.applyDiff({ - name: removeExtension(fileAsset.name), - video: stream, - mimeType: fileAsset.blob.type || "video/mp4", - }) - return assetId - } - - let image = await createImage(fileAsset.blob, { - owner: doc.$jazz.owner, - maxSize: 2048, - }) - existing.$jazz.applyDiff({ name: removeExtension(fileAsset.name), image }) - return assetId -} - -async function createAssetFromBlob( - assetFile: { name: string; blob: Blob }, - owner: Group, - now: Date, -) { - let isVideo = assetFile.blob.type.startsWith("video/") - if (isVideo) { - let video = await FileStream.createFromBlob(assetFile.blob, { - owner, - }) - return VideoAsset.create( - { - type: "video", - name: removeExtension(assetFile.name), - video, - mimeType: assetFile.blob.type || "video/mp4", - createdAt: now, - }, - owner, - ) - } - - let image = await createImage(assetFile.blob, { - owner, - maxSize: 2048, - }) - return ImageAsset.create( - { - type: "image", - name: removeExtension(assetFile.name), - image, - createdAt: now, - }, - owner, - ) -} - -function removeExtension(filename: string): string { - return filename.replace(/\.[^.]+$/, "") -} - -function isDocumentList(value: unknown): value is DocumentList { - if (typeof value !== "object" || value === null) return false - return "$jazz" in value && "find" in value -} - -function isSpaceBackupState(value: unknown): value is SpaceBackupState { - if (typeof value !== "object" || value === null) return false - if (!("directoryName" in value)) return false - return value.directoryName === null || typeof value.directoryName === "string" -} - -function findMovedManifestEntry( - manifest: Awaited>, - scannedByPath: Map, - matchedManifestDocIds: Set, - file: ScannedFile, - contentHash: string, -) { - if (!manifest) return null - let candidates = manifest.entries.filter(entry => { - if (matchedManifestDocIds.has(entry.docId)) return false - if (scannedByPath.has(entry.relativePath)) return false - if (entry.contentHash !== contentHash) return false - return true - }) - if (candidates.length === 0) return null - if (candidates.length === 1) return candidates[0] - - let matchingBasename = candidates.filter(entry => { - return getFilename(entry.relativePath) === getFilename(file.relativePath) - }) - if (matchingBasename.length === 1) { - return matchingBasename[0] - } - - return null -} - -function getFilename(relativePath: string): string { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length === 0) return relativePath - return parts[parts.length - 1] -} - -function getAssetFilesByIdFromManifest( - manifestEntry: ManifestEntry, -): Map { - let filesById = new Map() - for (let asset of manifestEntry.assets) { - if (!asset.id) continue - filesById.set(asset.id, asset.name) - } - return filesById -} - -function applyPathFromRelativePath( - content: string, - relativePath: string, - hasAssets: boolean, -): string { - let diskPath = derivePathFromRelativePath(relativePath, hasAssets) - let { frontmatter } = parseFrontmatter(content) - let currentPath = getPath(content) - - if (!frontmatter) { - if (!diskPath) return content - return `---\npath: ${diskPath}\n---\n\n${content}` - } - - if (currentPath === diskPath) return content - - if (currentPath && !diskPath) { - return content.replace( - /^(---\r?\n[\s\S]*?)path:\s*[^\r\n]*\r?\n([\s\S]*?---)/, - "$1$2", - ) - } - - if (currentPath && diskPath) { - return content.replace( - /^(---\r?\n[\s\S]*?)path:\s*[^\r\n]*/, - `$1path: ${diskPath}`, - ) - } - - if (!currentPath && diskPath) { - return content.replace(/^(---\r?\n)/, `$1path: ${diskPath}\n`) - } - - return content -} - -function derivePathFromRelativePath( - relativePath: string, - hasAssets: boolean, -): string | null { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length <= 1) return null - - let directoryParts = parts.slice(0, -1) - if (!hasAssets) { - let path = directoryParts.join("/") - return path || null - } - - let parentParts = directoryParts.slice(0, -1) - let path = parentParts.join("/") - return path || null -} - -function buildLocationFromRelativePath( - baseLocation: ReturnType extends Map< - string, - infer V - > - ? V - : never, - relativePath: string, -) { - let parts = relativePath.split("/").filter(Boolean) - if (parts.length === 0) return baseLocation - - return { - ...baseLocation, - dirPath: parts.slice(0, -1).join("/"), - filename: parts[parts.length - 1], - } -} - -function supportsFileSystemWatch(): boolean { - return typeof window.FileSystemObserver === "function" -} - -function isBackupSupported(): boolean { - return "showDirectoryPicker" in window -} - -function getSpaceBackupStorageKey(spaceId: string): string { - return `${SPACE_BACKUP_KEY_PREFIX}${spaceId}` -} - -function getSpacesWithBackup( - me: ReturnType< - typeof useAccount - >, - _storageVersion: number, -): string[] { - if (!me.$isLoaded || !me.root?.spaces?.$isLoaded) return [] - - let spaceIds: string[] = [] - for (let space of Array.from(me.root.spaces)) { - if (!space?.$isLoaded) continue - let backupPath = getSpaceBackupPath(space.$jazz.id) - if (backupPath) { - spaceIds.push(space.$jazz.id) - } - } - return spaceIds -} - -async function observeDirectoryChanges( - handle: FileSystemDirectoryHandle, - onChange: () => void, -): Promise<(() => void) | null> { - let Observer = window.FileSystemObserver - if (!Observer) return null - - let observer = new Observer(() => { - onChange() - }) - await observer.observe(handle, { recursive: true }) - return () => observer.disconnect() -} - -async function hashBlob(blob: Blob): Promise { - let buffer = await blob.arrayBuffer() - let hashBuffer = await crypto.subtle.digest("SHA-256", buffer) - let hashArray = Array.from(new Uint8Array(hashBuffer)) - return hashArray - .map(b => b.toString(16).padStart(2, "0")) - .join("") - .slice(0, 16) -} - -async function hashScannedAssets( - assets: { name: string; blob: Blob }[], -): Promise { - let hashed = await Promise.all( - assets.map(async asset => ({ - name: asset.name, - hash: await hashBlob(asset.blob), - })), - ) - return hashed -} - -function areScannedAssetsInSync( - manifestAssets: { id?: string; name: string; hash: string }[], - scannedAssets: ScannedAssetHash[], -): boolean { - if (manifestAssets.length !== scannedAssets.length) return false - - let sortedManifest = [...manifestAssets].sort((a, b) => - a.name.localeCompare(b.name), - ) - let sortedScanned = [...scannedAssets].sort((a, b) => - a.name.localeCompare(b.name), - ) - - for (let i = 0; i < sortedManifest.length; i++) { - if (sortedManifest[i].name !== sortedScanned[i].name) return false - if (sortedManifest[i].hash !== sortedScanned[i].hash) return false - } - - return true -} - -function areManifestAssetsEqual( - a: { id?: string; name: string; hash: string }[], - b: { id?: string; name: string; hash: string }[], -): boolean { - if (a.length !== b.length) return false - - let sortedA = [...a].sort((left, right) => - left.name.localeCompare(right.name), - ) - let sortedB = [...b].sort((left, right) => - left.name.localeCompare(right.name), - ) - - for (let i = 0; i < sortedA.length; i++) { - if ((sortedA[i].id ?? null) !== (sortedB[i].id ?? null)) return false - if (sortedA[i].name !== sortedB[i].name) return false - if (sortedA[i].hash !== sortedB[i].hash) return false - } - - return true -} - -function wasRecentlyImported(relativePath: string): boolean { - let importedAt = recentImportedRelativePaths.get(relativePath) - if (!importedAt) return false - if (Date.now() - importedAt > RECENT_IMPORT_WINDOW_MS) { - recentImportedRelativePaths.delete(relativePath) - return false - } - return true -} - -function markRecentlyImported(relativePath: string): void { - recentImportedRelativePaths.set(relativePath, Date.now()) -} - -function toTimestamp(value: string | null): number | null { - if (!value) return null - let ms = Date.parse(value) - return Number.isNaN(ms) ? null : ms -} - -function getDocLocationKey(doc: BackupDoc): string { - let path = doc.path ?? "" - let hasAssets = doc.assets.length > 0 ? "assets" : "no-assets" - return `${doc.title}|${path}|${hasAssets}` -} - -async function getSpaceBackupHandle( - spaceId: string, -): Promise { - try { - let handle = await idbGet( - `${HANDLE_STORAGE_KEY}-space-${spaceId}`, - ) - if (!handle) return null - let hasPermission = await verifyPermission(handle) - if (!hasPermission) return null - return handle - } catch { - return null - } -} diff --git a/src/main.tsx b/src/main.tsx index 2c4b58b..4203f4f 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -12,7 +12,10 @@ import { useSplashDelay, } from "./components/splash-screen" import { PWAContext, usePWAProvider, PWAInstallHint } from "./lib/pwa" -import { BackupSubscriber, SpacesBackupSubscriber } from "./lib/backup" +import { + BackupSubscriber, + SpacesBackupSubscriber, +} from "./lib/backup-subscribers" import { useCleanupDeleted } from "./lib/use-cleanup-deleted" import { init } from "@plausible-analytics/tracker" diff --git a/src/routes/settings.tsx b/src/routes/settings.tsx index 790633f..38c5291 100644 --- a/src/routes/settings.tsx +++ b/src/routes/settings.tsx @@ -59,7 +59,7 @@ import { import { wordlist } from "@/lib/wordlist" import { Footer } from "@/components/footer" import { usePWA, useIsPWAInstalled, PWAInstallDialog } from "@/lib/pwa" -import { BackupSettings } from "@/lib/backup" +import { BackupSettings } from "@/lib/backup-settings-ui" import { Tooltip, TooltipTrigger, diff --git a/src/routes/spaces.$spaceId.settings.tsx b/src/routes/spaces.$spaceId.settings.tsx index d530e41..3c2fa60 100644 --- a/src/routes/spaces.$spaceId.settings.tsx +++ b/src/routes/spaces.$spaceId.settings.tsx @@ -33,7 +33,7 @@ import { SpaceNotFound, SpaceUnauthorized, } from "@/components/document-error-states" -import { SpaceBackupSettings } from "@/lib/backup" +import { SpaceBackupSettings } from "@/lib/backup-settings-ui" import { getSpaceGroup, leaveSpace, From 5d5de9e69b4ef19fc75c9f0c0651aa4a5803ae42 Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Sun, 15 Feb 2026 23:18:02 +0100 Subject: [PATCH 14/19] include video assets in backups --- src/lib/backup-subscribers.test.ts | 13 +++++++++++++ src/lib/backup-subscribers.tsx | 19 ++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 src/lib/backup-subscribers.test.ts diff --git a/src/lib/backup-subscribers.test.ts b/src/lib/backup-subscribers.test.ts new file mode 100644 index 0000000..30897b9 --- /dev/null +++ b/src/lib/backup-subscribers.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest" +import { spaceBackupDocumentResolve } from "./backup-subscribers" + +describe("space backup subscriber query", () => { + it("resolves image and video assets", () => { + expect(spaceBackupDocumentResolve.documents.$each.assets.$each.image).toBe( + true, + ) + expect(spaceBackupDocumentResolve.documents.$each.assets.$each.video).toBe( + true, + ) + }) +}) diff --git a/src/lib/backup-subscribers.tsx b/src/lib/backup-subscribers.tsx index c4de442..e576b5b 100644 --- a/src/lib/backup-subscribers.tsx +++ b/src/lib/backup-subscribers.tsx @@ -22,7 +22,17 @@ import { toTimestamp, } from "@/lib/backup-storage" -export { BackupSubscriber, SpacesBackupSubscriber } +export { BackupSubscriber, SpacesBackupSubscriber, spaceBackupDocumentResolve } + +let spaceBackupDocumentResolve = { + documents: { + $each: { + content: true, + assets: { $each: { image: true, video: true } }, + }, + $onError: "catch", + }, +} as const let backupQuery = { root: { @@ -205,12 +215,7 @@ function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { let isPullingRef = useRef(false) let space = useCoState(Space, spaceId, { - resolve: { - documents: { - $each: { content: true, assets: { $each: { image: true } } }, - $onError: "catch", - }, - }, + resolve: spaceBackupDocumentResolve, }) useEffect(() => { From c8a519fac2cfd067c4050c79cae29bbf5af8d3c8 Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Sun, 15 Feb 2026 23:23:55 +0100 Subject: [PATCH 15/19] cover more edge cases --- src/lib/backup-engine.ts | 39 +++++++++++++++++++++++++------- src/lib/backup-scenarios.test.ts | 27 ++++++++++++++++++++++ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/src/lib/backup-engine.ts b/src/lib/backup-engine.ts index 5ac2c60..7cd7a91 100644 --- a/src/lib/backup-engine.ts +++ b/src/lib/backup-engine.ts @@ -47,6 +47,8 @@ interface ScannedAssetHash { let preferredRelativePathByDocId = new Map() let recentImportedRelativePaths = new Map() let RECENT_IMPORT_WINDOW_MS = 30_000 +let handleScopeIds = new WeakMap() +let handleScopeCounter = 0 async function hashContent(content: string): Promise { let encoder = new TextEncoder() @@ -82,6 +84,7 @@ async function syncFromBackup( let manifest = await readManifest(handle) let scannedFiles = await scanBackupFolder(handle) let listOwner = targetDocs.$jazz.owner + let importScopeKey = getHandleScopeKey(handle) let manifestByPath = new Map( manifest?.entries.map(e => [e.relativePath, e]) ?? [], @@ -134,10 +137,10 @@ async function syncFromBackup( result.errors.push(`Cannot create ${file.name}: no write permission`) continue } - if (wasRecentlyImported(file.relativePath)) continue + if (wasRecentlyImported(importScopeKey, file.relativePath)) continue let newDocId = await createDocFromFile(file, targetDocs, listOwner) preferredRelativePathByDocId.set(newDocId, file.relativePath) - markRecentlyImported(file.relativePath) + markRecentlyImported(importScopeKey, file.relativePath) result.created++ } else if ( manifestEntry.contentHash !== contentHash || @@ -280,6 +283,7 @@ async function performSyncBackup( docs: BackupDoc[], ): Promise { let docLocations = computeDocLocations(docs) + let importScopeKey = getHandleScopeKey(handle) let existingManifest = await readManifest(handle) let existingEntriesByDocId = new Map( existingManifest?.entries.map(entry => [entry.docId, entry]) ?? [], @@ -378,7 +382,9 @@ async function performSyncBackup( }) for (let entry of manifestEntries) { - recentImportedRelativePaths.delete(entry.relativePath) + recentImportedRelativePaths.delete( + makeRecentImportKey(importScopeKey, entry.relativePath), + ) } } @@ -950,18 +956,35 @@ function areManifestAssetsEqual( return true } -function wasRecentlyImported(relativePath: string): boolean { - let importedAt = recentImportedRelativePaths.get(relativePath) +function wasRecentlyImported(scopeKey: string, relativePath: string): boolean { + let key = makeRecentImportKey(scopeKey, relativePath) + let importedAt = recentImportedRelativePaths.get(key) if (!importedAt) return false if (Date.now() - importedAt > RECENT_IMPORT_WINDOW_MS) { - recentImportedRelativePaths.delete(relativePath) + recentImportedRelativePaths.delete(key) return false } return true } -function markRecentlyImported(relativePath: string): void { - recentImportedRelativePaths.set(relativePath, Date.now()) +function markRecentlyImported(scopeKey: string, relativePath: string): void { + recentImportedRelativePaths.set( + makeRecentImportKey(scopeKey, relativePath), + Date.now(), + ) +} + +function makeRecentImportKey(scopeKey: string, relativePath: string): string { + return `${scopeKey}:${relativePath}` +} + +function getHandleScopeKey(handle: FileSystemDirectoryHandle): string { + let existing = handleScopeIds.get(handle) + if (existing) return existing + handleScopeCounter += 1 + let created = `backup-handle-${handleScopeCounter}` + handleScopeIds.set(handle, created) + return created } function getDocLocationKey(doc: BackupDoc): string { diff --git a/src/lib/backup-scenarios.test.ts b/src/lib/backup-scenarios.test.ts index a41601b..2c678b9 100644 --- a/src/lib/backup-scenarios.test.ts +++ b/src/lib/backup-scenarios.test.ts @@ -72,6 +72,33 @@ describe("backup scenarios", () => { expect(imported).toBeDefined() }) + it("imports same relative path across different backup roots", async () => { + let firstInitialCount = getLoadedDocs(docs).length + let firstRoot = new MockDirectoryHandle("first-root") + let secondRoot = new MockDirectoryHandle("second-root") + firstRoot.addFile( + "Cross Scope Note.md", + "# Cross Scope Note\n\nfrom first root", + 2_000, + ) + secondRoot.addFile( + "Cross Scope Note.md", + "# Cross Scope Note\n\nfrom second root", + 2_000, + ) + + let secondDocs = co.list(Document).create([], Group.create()) + let secondInitialCount = getLoadedDocs(secondDocs).length + + let firstResult = await syncFromBackup(firstRoot, docs, true) + let secondResult = await syncFromBackup(secondRoot, secondDocs, true) + + expect(firstResult.created).toBe(1) + expect(secondResult.created).toBe(1) + expect(getLoadedDocs(docs)).toHaveLength(firstInitialCount + 1) + expect(getLoadedDocs(secondDocs)).toHaveLength(secondInitialCount + 1) + }) + it("imports asset references with created asset ids", async () => { await writeFileAtPath( root, From baf270621d5c1fa1e87d0b61cf920570fc7ba3ff Mon Sep 17 00:00:00 2001 From: Carl Assmann Date: Sun, 15 Feb 2026 23:24:56 +0100 Subject: [PATCH 16/19] fix typos --- src/lib/editor-utils.ts | 2 +- src/lib/pwa.tsx | 8 ++++---- src/routes/settings.tsx | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/lib/editor-utils.ts b/src/lib/editor-utils.ts index 78d4074..1b4e9c6 100644 --- a/src/lib/editor-utils.ts +++ b/src/lib/editor-utils.ts @@ -291,7 +291,7 @@ function setupKeyboardShortcuts(opts: { } function showAutosaveToast() { - toast("Alkalyte saves automatically", { + toast("Alkalye saves automatically", { description: "Changes are saved locally and synced to the cloud while you type.", action: { diff --git a/src/lib/pwa.tsx b/src/lib/pwa.tsx index 95a81d1..d5b9415 100644 --- a/src/lib/pwa.tsx +++ b/src/lib/pwa.tsx @@ -153,7 +153,7 @@ function PWAInstallHint() { toast(
-
Install Alkalyte
+
Install Alkalye
Add to your homescreen for the best experience.
@@ -213,11 +213,11 @@ function PWAInstallDialog({ - Install Alkalyte + Install Alkalye {isMobileDevice() - ? "Add Alkalyte to your homescreen for instant access and the best experience." - : "Install Alkalyte as an app for quick access and a better experience."} + ? "Add Alkalye to your homescreen for instant access and the best experience." + : "Install Alkalye as an app for quick access and a better experience."}
diff --git a/src/routes/settings.tsx b/src/routes/settings.tsx index 38c5291..80f10ac 100644 --- a/src/routes/settings.tsx +++ b/src/routes/settings.tsx @@ -1046,7 +1046,7 @@ function InstallationSection() { Not installed

- Install Alkalyte to your device for the best experience. + Install Alkalye to your device for the best experience.

- Folder: {directoryName} + Folder:{" "} + + {directoryName} +

{formattedLastBackup && (

@@ -112,27 +121,33 @@ function BackupSettings() {

)}
-
+ setBidirectional(e.target.checked)} - disabled={!supportsFileSystemWatch()} - className="size-4 rounded border-gray-300" + onCheckedChange={setBidirectional} + disabled={!canWatchFileSystem || isLoading} /> - Sync changes from folder - -

- {supportsFileSystemWatch() - ? "When enabled, changes made in the backup folder will be imported into Alkalye." - : "Requires a Chromium-based browser with File System Observer support."} -

+
@@ -168,7 +183,9 @@ function BackupSettings() { disabled={isLoading} > - Choose backup folder + {pendingAction === "enable" + ? "Choosing..." + : "Choose backup folder"} )} @@ -184,59 +201,60 @@ interface SpaceBackupSettingsProps { function SpaceBackupSettings({ spaceId, isAdmin }: SpaceBackupSettingsProps) { let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) - let [isLoading, setIsLoading] = useState(false) + let [pendingAction, setPendingAction] = useState< + "choose" | "change" | "clear" | null + >(null) + let [error, setError] = useState(null) + let isLoading = pendingAction !== null if (!isBackupSupported()) { - return ( -
-

- Local Backup -

-
-
- -
-

- Local backup requires a Chromium-based browser (Chrome, Edge, - Brave, or Opera). -

-

- Safari and Firefox do not support the File System Access API - needed for this feature. -

-
-
-
-
- ) + return } async function handleChooseFolder() { - setIsLoading(true) + setPendingAction("choose") + setError(null) try { let handle = await window.showDirectoryPicker({ mode: "readwrite" }) await setSpaceBackupHandle(spaceId, handle) setDirectoryName(handle.name) } catch (e) { if (!(e instanceof Error && e.name === "AbortError")) { + setError("Failed to choose folder. Try again.") console.error("Failed to select folder:", e) } } finally { - setIsLoading(false) + setPendingAction(null) } } async function handleChangeFolder() { - await handleChooseFolder() + setPendingAction("change") + setError(null) + try { + let handle = await window.showDirectoryPicker({ mode: "readwrite" }) + await setSpaceBackupHandle(spaceId, handle) + setDirectoryName(handle.name) + } catch (e) { + if (!(e instanceof Error && e.name === "AbortError")) { + setError("Failed to choose folder. Try again.") + console.error("Failed to select folder:", e) + } + } finally { + setPendingAction(null) + } } async function handleClear() { - setIsLoading(true) + setPendingAction("clear") + setError(null) try { await clearSpaceBackupHandle(spaceId) setDirectoryName(null) + } catch { + setError("Failed to clear folder. Try again.") } finally { - setIsLoading(false) + setPendingAction(null) } } @@ -253,8 +271,20 @@ function SpaceBackupSettings({ spaceId, isAdmin }: SpaceBackupSettingsProps) { Backup folder set

- Folder: {directoryName} + Folder:{" "} + + {directoryName} +

+ {error && ( +
+ + {error} +
+ )}
+ {!isAdmin && ( +

+ Only space admins can change this folder. +

+ )} ) : ( <> @@ -282,6 +319,12 @@ function SpaceBackupSettings({ spaceId, isAdmin }: SpaceBackupSettingsProps) {

Set a backup folder for this space's documents.

+ {error && ( +
+ + {error} +
+ )} + {!isAdmin && ( +

+ Only space admins can set a backup folder. +

+ )} )}

) } + +function UnsupportedBrowserCallout() { + return ( +
+

+ Local Backup +

+
+
+ +
+

+ Local backup requires a Chromium-based browser (Chrome, Edge, + Brave, or Opera). +

+

+ Safari and Firefox do not support the File System Access API + needed for this feature. +

+
+
+
+
+ ) +}