diff --git a/src/lib/backup-engine.ts b/src/lib/backup-engine.ts new file mode 100644 index 0000000..53d387b --- /dev/null +++ b/src/lib/backup-engine.ts @@ -0,0 +1,1234 @@ +import { co, Group, Account, FileStream } from "jazz-tools" +import { createImage } from "jazz-tools/media" +import { Document, Asset, ImageAsset, VideoAsset } from "@/schema" +import { getDocumentTitle } from "@/lib/document-utils" +import { getPath, parseFrontmatter } from "@/editor/frontmatter" +import { + computeDocLocations, + transformContentForBackup, + computeExpectedStructure, + scanBackupFolder, + readManifest, + writeManifest, + transformContentForImport, + type BackupDoc, + type ManifestEntry, + type ScannedFile, +} from "@/lib/backup-sync" + +export { + hashContent, + syncBackup, + syncFromBackup, + prepareBackupDoc, + type LoadedDocument, + type DocumentList, +} + +type LoadedDocument = co.loaded< + typeof Document, + { content: true; assets: { $each: { image: true; video: true } } } +> + +type DocumentList = co.loaded>> + +interface SyncFromBackupResult { + created: number + updated: number + deleted: number + errors: string[] +} + +interface ScannedAssetHash { + name: string + hash: string +} + +let preferredRelativePathByDocId = new Map() +let recentImportedRelativePaths = new Map() +let RECENT_IMPORT_WINDOW_MS = 30_000 + +async function hashContent(content: string): Promise { + let encoder = new TextEncoder() + let data = encoder.encode(content) + let hashBuffer = await crypto.subtle.digest("SHA-256", data) + let hashArray = Array.from(new Uint8Array(hashBuffer)) + return hashArray + .map(b => b.toString(16).padStart(2, "0")) + .join("") + .slice(0, 16) +} + +async function syncBackup( + handle: FileSystemDirectoryHandle, + docs: BackupDoc[], + scopeId = "docs:unknown", +): Promise { + await performSyncBackup(handle, docs, scopeId) +} + +async function syncFromBackup( + handle: FileSystemDirectoryHandle, + targetDocs: DocumentList, + canWrite: boolean, + lastPullAtMs: number | null = null, +): Promise { + let result: SyncFromBackupResult = { + created: 0, + updated: 0, + deleted: 0, + errors: [], + } + + let manifest = await readManifest(handle) + let scannedFiles = await scanBackupFolder(handle) + let listOwner = targetDocs.$jazz.owner + let importScopeKey = getDocumentListScopeKey(targetDocs) + let manifestEntriesForScope = + manifest?.entries.filter(entry => + manifestEntryMatchesScope(entry, importScopeKey), + ) ?? [] + let manifestEntriesOutsideScope = + manifest?.entries.filter( + entry => !manifestEntryMatchesScope(entry, importScopeKey), + ) ?? [] + let nextManifestByDocId = new Map( + manifestEntriesForScope.map(entry => [entry.docId, entry]), + ) + let manifestChanged = false + + let manifestByPath = new Map( + manifestEntriesForScope.map(e => [e.relativePath, e]), + ) + let scannedByPath = new Map(scannedFiles.map(f => [f.relativePath, f])) + let matchedManifestDocIds = new Set() + + for (let file of scannedFiles) { + try { + let contentHash = await hashContent(file.content) + let scannedAssetHashes = await hashScannedAssets(file.assets) + let manifestEntry = manifestByPath.get(file.relativePath) + let assetsInSync = false + if (manifestEntry) { + matchedManifestDocIds.add(manifestEntry.docId) + assetsInSync = areScannedAssetsInSync( + manifestEntry.assets, + scannedAssetHashes, + ) + let unchangedSinceManifest = + manifestEntry.contentHash === contentHash && + manifestEntry.relativePath === file.relativePath && + assetsInSync + let isBeforeLastPull = + lastPullAtMs !== null && file.lastModified < lastPullAtMs + if (isBeforeLastPull && unchangedSinceManifest) continue + } + + if (!manifestEntry) { + let movedEntry = findMovedManifestEntry( + manifestEntriesForScope, + scannedByPath, + matchedManifestDocIds, + file, + contentHash, + scannedAssetHashes, + ) + if (movedEntry) { + manifestEntry = movedEntry + matchedManifestDocIds.add(movedEntry.docId) + assetsInSync = areScannedAssetsInSync( + movedEntry.assets, + scannedAssetHashes, + ) + } + } + + if (!manifestEntry) { + let matchingUntrackedDocId = findMatchingUntrackedDocId( + file, + targetDocs, + ) + if (matchingUntrackedDocId) { + let changed = upsertManifestEntry(nextManifestByDocId, { + docId: matchingUntrackedDocId, + relativePath: file.relativePath, + scopeId: importScopeKey, + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: buildManifestAssetsFromScanned(scannedAssetHashes), + }) + manifestChanged = manifestChanged || changed + preferredRelativePathByDocId.set( + matchingUntrackedDocId, + file.relativePath, + ) + continue + } + + if (!canWrite) { + result.errors.push(`Cannot create ${file.name}: no write permission`) + continue + } + if (wasRecentlyImported(importScopeKey, file.relativePath)) continue + let newDocId = await createDocFromFile(file, targetDocs, listOwner) + let changed = upsertManifestEntry(nextManifestByDocId, { + docId: newDocId, + relativePath: file.relativePath, + scopeId: importScopeKey, + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: buildManifestAssetsFromScanned(scannedAssetHashes), + }) + manifestChanged = manifestChanged || changed + preferredRelativePathByDocId.set(newDocId, file.relativePath) + markRecentlyImported(importScopeKey, file.relativePath) + result.created++ + } else if ( + manifestEntry.contentHash !== contentHash || + manifestEntry.relativePath !== file.relativePath || + !assetsInSync + ) { + preferredRelativePathByDocId.set(manifestEntry.docId, file.relativePath) + if (!canWrite) { + result.errors.push(`Cannot update ${file.name}: no write permission`) + continue + } + let didUpdate = await updateDocFromFile( + file, + manifestEntry.docId, + manifestEntry, + targetDocs, + ) + if (didUpdate) { + let changed = upsertManifestEntry(nextManifestByDocId, { + docId: manifestEntry.docId, + relativePath: file.relativePath, + scopeId: importScopeKey, + locationKey: manifestEntry.locationKey, + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: buildManifestAssetsFromScanned( + scannedAssetHashes, + manifestEntry.assets, + ), + }) + manifestChanged = manifestChanged || changed + result.updated++ + } else { + result.errors.push( + `Skipped update for ${file.relativePath}: target document not loaded`, + ) + } + } else if (manifestEntry) { + let changed = upsertManifestEntry(nextManifestByDocId, { + docId: manifestEntry.docId, + relativePath: file.relativePath, + scopeId: importScopeKey, + locationKey: manifestEntry.locationKey, + contentHash, + lastSyncedAt: manifestEntry.lastSyncedAt, + assets: buildManifestAssetsFromScanned( + scannedAssetHashes, + manifestEntry.assets, + ), + }) + manifestChanged = manifestChanged || changed + } + } catch (err) { + result.errors.push( + `Failed to process ${file.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, + ) + } + } + + if (manifestEntriesForScope.length > 0 && canWrite) { + for (let entry of manifestEntriesForScope) { + if (matchedManifestDocIds.has(entry.docId)) continue + if (!scannedByPath.has(entry.relativePath)) { + try { + let doc = targetDocs.find(d => d?.$jazz.id === entry.docId) + if (doc?.$isLoaded && !doc.deletedAt) { + doc.$jazz.set("deletedAt", new Date()) + doc.$jazz.set("updatedAt", new Date()) + result.deleted++ + } + nextManifestByDocId.delete(entry.docId) + manifestChanged = true + } catch (err) { + result.errors.push( + `Failed to delete ${entry.relativePath}: ${err instanceof Error ? err.message : "Unknown error"}`, + ) + } + } + } + } + + let nextScopeEntries = Array.from(nextManifestByDocId.values()) + if ( + manifestChanged || + haveManifestEntriesChanged(manifestEntriesForScope, nextScopeEntries) + ) { + await writeManifest(handle, { + version: 1, + entries: [...nextScopeEntries, ...manifestEntriesOutsideScope], + lastSyncAt: new Date().toISOString(), + }) + } + + return result +} + +async function prepareBackupDoc(doc: LoadedDocument): Promise { + let content = doc.content?.toString() ?? "" + let title = getDocumentTitle(doc) + let path = getPath(content) + let updatedAtMs = doc.updatedAt?.getTime() ?? 0 + + let assets: BackupDoc["assets"] = [] + if (doc.assets?.$isLoaded) { + for (let asset of [...doc.assets]) { + if (!asset?.$isLoaded) continue + + let blob: Blob | undefined + if (asset.type === "image" && asset.image?.$isLoaded) { + let original = asset.image.original + if (original?.$isLoaded) { + blob = original.toBlob() + } + } else if (asset.type === "video" && asset.video?.$isLoaded) { + blob = await asset.video.toBlob() + } + + if (blob) { + assets.push({ id: asset.$jazz.id, name: asset.name, blob }) + } + } + } + + return { id: doc.$jazz.id, title, content, path, updatedAtMs, assets } +} + +async function getOrCreateDirectory( + parent: FileSystemDirectoryHandle, + path: string, +): Promise { + let parts = path.split("/").filter(Boolean) + let current = parent + for (let part of parts) { + current = await current.getDirectoryHandle(part, { create: true }) + } + return current +} + +async function writeFile( + dir: FileSystemDirectoryHandle, + name: string, + content: string | Blob, +): Promise { + let file = await dir.getFileHandle(name, { create: true }) + let writable = await file.createWritable() + await writable.write(content) + await writable.close() +} + +async function deleteFile( + dir: FileSystemDirectoryHandle, + name: string, +): Promise { + try { + await dir.removeEntry(name) + } catch { + return + } +} + +async function listFiles(dir: FileSystemDirectoryHandle): Promise { + let files: string[] = [] + for await (let [name, handle] of dir.entries()) { + if (handle.kind === "file") files.push(name) + } + return files +} + +async function listDirectories( + dir: FileSystemDirectoryHandle, +): Promise { + let dirs: string[] = [] + for await (let [name, handle] of dir.entries()) { + if (handle.kind === "directory") dirs.push(name) + } + return dirs +} + +async function performSyncBackup( + handle: FileSystemDirectoryHandle, + docs: BackupDoc[], + scopeId: string, +): Promise { + let docLocations = computeDocLocations(docs) + let existingManifest = await readManifest(handle) + let existingScopeEntries = + existingManifest?.entries.filter(entry => + manifestEntryMatchesScope(entry, scopeId), + ) ?? [] + let existingForeignEntries = + existingManifest?.entries.filter( + entry => !manifestEntryMatchesScope(entry, scopeId), + ) ?? [] + let existingEntriesByDocId = new Map( + existingScopeEntries.map(entry => [entry.docId, entry]), + ) + let manifestEntries: { + docId: string + relativePath: string + scopeId: string + locationKey: string + contentHash: string + lastSyncedAt: string + assets: { id: string; name: string; hash: string }[] + }[] = [] + let hasFilesystemChanges = false + let nowIso = new Date().toISOString() + + for (let doc of docs) { + let loc = docLocations.get(doc.id)! + let locationKey = getDocLocationKey(doc) + let computedRelativePath = loc.dirPath + ? `${loc.dirPath}/${loc.filename}` + : loc.filename + let existingEntry = existingEntriesByDocId.get(doc.id) + let preferredRelativePath = preferredRelativePathByDocId.get(doc.id) + let finalRelativePath = computedRelativePath + if (existingEntry) { + if (existingEntry.locationKey === locationKey) { + finalRelativePath = existingEntry.relativePath + } + } + if (preferredRelativePath) { + finalRelativePath = preferredRelativePath + } + let finalLocation = buildLocationFromRelativePath(loc, finalRelativePath) + docLocations.set(doc.id, finalLocation) + loc = finalLocation + + let dir = loc.dirPath + ? await getOrCreateDirectory(handle, loc.dirPath) + : handle + + let exportedContent = transformContentForBackup(doc.content, loc.assetFiles) + let contentHash = await hashContent(exportedContent) + let relativePath = finalRelativePath + let assets: { id: string; name: string; hash: string }[] = [] + for (let asset of doc.assets) { + let filename = loc.assetFiles.get(asset.id)! + assets.push({ + id: asset.id, + name: filename, + hash: await hashBlob(asset.blob), + }) + } + + let shouldWriteDoc = + !existingEntry || + existingEntry.relativePath !== relativePath || + existingEntry.contentHash !== contentHash || + !areManifestAssetsEqual(existingEntry.assets, assets) + + if (!shouldWriteDoc) { + let docFileExists = await fileExists(dir, loc.filename) + if (!docFileExists) { + shouldWriteDoc = true + } else if (doc.assets.length > 0) { + let assetsExist = await assetsExistAtLocation(dir, loc, doc) + if (!assetsExist) { + shouldWriteDoc = true + } + } + } + + if (shouldWriteDoc) { + hasFilesystemChanges = true + await writeFile(dir, loc.filename, exportedContent) + + if (doc.assets.length > 0) { + let assetsDir = await dir.getDirectoryHandle("assets", { create: true }) + for (let asset of doc.assets) { + let filename = loc.assetFiles.get(asset.id)! + await writeFile(assetsDir, filename, asset.blob) + } + } + } + + manifestEntries.push({ + docId: doc.id, + relativePath, + scopeId, + locationKey, + contentHash, + lastSyncedAt: shouldWriteDoc + ? nowIso + : (existingEntry?.lastSyncedAt ?? nowIso), + assets, + }) + } + + let currentDocIds = new Set(docs.map(doc => doc.id)) + let shouldPreserveMissingScopeEntries = + scopeId !== "docs:unknown" && docs.length < existingEntriesByDocId.size + let preservedScopeEntries = shouldPreserveMissingScopeEntries + ? existingScopeEntries.filter(entry => !currentDocIds.has(entry.docId)) + : [] + let nextScopeEntries = [...manifestEntries, ...preservedScopeEntries] + let docsChanged = + existingScopeEntries.length !== nextScopeEntries.length || + hasFilesystemChanges + let shouldCleanup = + existingForeignEntries.length === 0 && !shouldPreserveMissingScopeEntries + + if (docsChanged) { + if (shouldCleanup) { + await cleanupOrphanedFiles(handle, docs, docLocations) + } + + await writeManifest(handle, { + version: 1, + entries: [...nextScopeEntries, ...existingForeignEntries], + lastSyncAt: nowIso, + }) + } + + for (let doc of docs) { + preferredRelativePathByDocId.delete(doc.id) + } +} + +async function cleanupOrphanedFiles( + handle: FileSystemDirectoryHandle, + docs: BackupDoc[], + docLocations: Map< + string, + ReturnType extends Map + ? V + : never + >, +): Promise { + let { expectedPaths, expectedFiles } = computeExpectedStructure( + docs, + docLocations, + ) + let expectedAssetFilesByDir = new Map>() + for (let doc of docs) { + let location = docLocations.get(doc.id) + if (!location || !location.hasOwnFolder) continue + let assetsPath = location.dirPath ? `${location.dirPath}/assets` : "assets" + expectedAssetFilesByDir.set( + assetsPath, + new Set(location.assetFiles.values()), + ) + } + + async function cleanAssetsDirectory( + dir: FileSystemDirectoryHandle, + path: string, + ): Promise { + let hasContent = false + let expected = expectedAssetFilesByDir.get(path) ?? new Set() + + for await (let [name, child] of dir.entries()) { + if (name.startsWith(".")) continue + if (child.kind === "directory") { + await dir.removeEntry(name, { recursive: true }) + continue + } + if (expected.has(name)) { + hasContent = true + continue + } + await deleteFile(dir, name) + } + + return hasContent + } + + async function cleanDir( + dir: FileSystemDirectoryHandle, + path: string, + ): Promise { + let subdirs = await listDirectories(dir) + let hasContent = false + + for (let subdir of subdirs) { + let subPath = path ? `${path}/${subdir}` : subdir + + if (subdir === "assets" && expectedPaths.has(subPath)) { + let subHandle = await dir.getDirectoryHandle(subdir) + let assetsHasContent = await cleanAssetsDirectory(subHandle, subPath) + if (assetsHasContent) hasContent = true + continue + } + + if (expectedPaths.has(subPath)) { + let subHandle = await dir.getDirectoryHandle(subdir) + let subHasContent = await cleanDir(subHandle, subPath) + if (subHasContent) hasContent = true + } else { + try { + await dir.removeEntry(subdir, { recursive: true }) + } catch { + continue + } + } + } + + let expected = expectedFiles.get(path) ?? new Set() + let files = await listFiles(dir) + for (let file of files) { + if (file.endsWith(".md")) { + if (expected.has(file)) { + hasContent = true + } else { + await deleteFile(dir, file) + } + } + } + + return hasContent + } + + await cleanDir(handle, "") +} + +async function createDocFromFile( + file: ScannedFile, + targetDocs: DocumentList, + listOwner: Group | Account, +): Promise { + let docGroup = Group.create() + if (listOwner instanceof Group) { + docGroup.addMember(listOwner) + } + + let now = new Date() + + let docAssets: co.loaded[] = [] + let assetFilesById = new Map() + for (let assetFile of file.assets) { + let asset = await createAssetFromBlob(assetFile, docGroup, now) + docAssets.push(asset) + assetFilesById.set(asset.$jazz.id, assetFile.name) + } + + let transformedContent = transformContentForImport( + file.content, + assetFilesById, + ) + let content = applyPathFromRelativePath( + transformedContent, + file.relativePath, + file.assets.length > 0, + ) + + let newDoc = Document.create( + { + version: 1, + content: co.plainText().create(content, docGroup), + assets: + docAssets.length > 0 + ? co.list(Asset).create(docAssets, docGroup) + : undefined, + createdAt: now, + updatedAt: now, + }, + docGroup, + ) + + targetDocs.$jazz.push(newDoc) + return newDoc.$jazz.id +} + +async function updateDocFromFile( + file: ScannedFile, + docId: string, + manifestEntry: ManifestEntry, + targetDocs: DocumentList, +): Promise { + let doc = targetDocs.find( + (d): d is LoadedDocument => d?.$isLoaded === true && d.$jazz.id === docId, + ) + if (!doc || !doc.content?.$isLoaded) { + return false + } + + let assetFilesById = await syncDocAssetsFromFile(doc, file, manifestEntry) + let content = applyPathFromRelativePath( + transformContentForImport(file.content, assetFilesById), + file.relativePath, + file.assets.length > 0, + ) + + doc.content.$jazz.applyDiff(content) + doc.$jazz.set("updatedAt", new Date()) + return true +} + +async function syncDocAssetsFromFile( + doc: LoadedDocument, + file: ScannedFile, + manifestEntry: ManifestEntry, +): Promise> { + if (file.assets.length === 0) { + let manifestFilesById = getAssetFilesByIdFromManifest(manifestEntry) + let keepsManifestRefs = Array.from(manifestFilesById.values()).some( + filename => file.content.includes(`assets/${filename}`), + ) + if (keepsManifestRefs) { + return manifestFilesById + } + if (doc.assets?.$isLoaded) { + for (let i = doc.assets.length - 1; i >= 0; i--) { + doc.assets.$jazz.splice(i, 1) + } + } + return new Map() + } + + if (!doc.assets) { + doc.$jazz.set("assets", co.list(Asset).create([], doc.$jazz.owner)) + } + if (!doc.assets?.$isLoaded) { + return getAssetFilesByIdFromManifest(manifestEntry) + } + + let currentAssets = Array.from(doc.assets).filter( + (asset): asset is co.loaded => asset?.$isLoaded === true, + ) + let assetsById = new Map(currentAssets.map(asset => [asset.$jazz.id, asset])) + let fileAssetsWithHash = await Promise.all( + file.assets.map(async asset => ({ + name: asset.name, + blob: asset.blob, + hash: await hashBlob(asset.blob), + })), + ) + + let manifestByName = new Map( + manifestEntry.assets.map(asset => [asset.name, asset]), + ) + let manifestByHash = new Map() + for (let asset of manifestEntry.assets) { + if (!manifestByHash.has(asset.hash)) { + manifestByHash.set(asset.hash, []) + } + manifestByHash.get(asset.hash)?.push(asset) + } + + let keepAssetIds = new Set() + let assetFilesById = new Map() + + for (let fileAsset of fileAssetsWithHash) { + let matchedByName = manifestByName.get(fileAsset.name) + let matchedId = + matchedByName?.id && assetsById.has(matchedByName.id) + ? matchedByName.id + : null + + if (!matchedId) { + let byHash = manifestByHash.get(fileAsset.hash) ?? [] + for (let candidate of byHash) { + if (!candidate.id || keepAssetIds.has(candidate.id)) continue + if (!assetsById.has(candidate.id)) continue + matchedId = candidate.id + break + } + } + + if (matchedId) { + let existing = assetsById.get(matchedId) + if (!existing) continue + + let shouldUpdateBinary = + matchedByName?.id === matchedId + ? matchedByName.hash !== fileAsset.hash + : false + + if (shouldUpdateBinary) { + matchedId = await syncExistingAssetFromFile( + doc, + existing, + matchedId, + fileAsset, + ) + } + + let updatedAsset = doc.assets.find( + asset => asset?.$isLoaded && asset.$jazz.id === matchedId, + ) + if (!updatedAsset) continue + assetsById.set(matchedId, updatedAsset) + if (updatedAsset.name !== removeExtension(fileAsset.name)) { + updatedAsset.$jazz.applyDiff({ name: removeExtension(fileAsset.name) }) + } + + keepAssetIds.add(matchedId) + assetFilesById.set(matchedId, fileAsset.name) + continue + } + + let created = await createAssetFromBlob( + fileAsset, + doc.$jazz.owner, + new Date(), + ) + doc.assets.$jazz.push(created) + keepAssetIds.add(created.$jazz.id) + assetFilesById.set(created.$jazz.id, fileAsset.name) + } + + for (let i = doc.assets.length - 1; i >= 0; i--) { + let asset = doc.assets[i] + if (!asset?.$isLoaded) continue + if (keepAssetIds.has(asset.$jazz.id)) continue + doc.assets.$jazz.splice(i, 1) + } + + if (fileAssetsWithHash.length === 0) { + return new Map() + } + + return assetFilesById +} + +async function syncExistingAssetFromFile( + doc: LoadedDocument, + existing: co.loaded, + assetId: string, + fileAsset: { name: string; blob: Blob }, +): Promise { + let nextType = fileAsset.blob.type.startsWith("video/") ? "video" : "image" + if (existing.type !== nextType) { + let index = + doc.assets?.findIndex(asset => asset?.$jazz.id === assetId) ?? -1 + if (index === -1 || !doc.assets) return assetId + doc.assets.$jazz.splice(index, 1) + let replacement = await createAssetFromBlob( + fileAsset, + doc.$jazz.owner, + existing.createdAt, + ) + doc.assets.$jazz.push(replacement) + return replacement.$jazz.id + } + + if (existing.type === "video") { + let stream = await FileStream.createFromBlob(fileAsset.blob, { + owner: doc.$jazz.owner, + }) + existing.$jazz.applyDiff({ + name: removeExtension(fileAsset.name), + video: stream, + mimeType: fileAsset.blob.type || "video/mp4", + }) + return assetId + } + + let image = await createImage(fileAsset.blob, { + owner: doc.$jazz.owner, + maxSize: 2048, + }) + existing.$jazz.applyDiff({ name: removeExtension(fileAsset.name), image }) + return assetId +} + +async function createAssetFromBlob( + assetFile: { name: string; blob: Blob }, + owner: Group, + now: Date, +) { + let isVideo = assetFile.blob.type.startsWith("video/") + if (isVideo) { + let video = await FileStream.createFromBlob(assetFile.blob, { + owner, + }) + return VideoAsset.create( + { + type: "video", + name: removeExtension(assetFile.name), + video, + mimeType: assetFile.blob.type || "video/mp4", + createdAt: now, + }, + owner, + ) + } + + let image = await createImage(assetFile.blob, { + owner, + maxSize: 2048, + }) + return ImageAsset.create( + { + type: "image", + name: removeExtension(assetFile.name), + image, + createdAt: now, + }, + owner, + ) +} + +function removeExtension(filename: string): string { + return filename.replace(/\.[^.]+$/, "") +} + +function findMovedManifestEntry( + manifestEntries: ManifestEntry[], + scannedByPath: Map, + matchedManifestDocIds: Set, + file: ScannedFile, + contentHash: string, + scannedAssetHashes: ScannedAssetHash[], +) { + let candidates = manifestEntries.filter(entry => { + if (matchedManifestDocIds.has(entry.docId)) return false + if (scannedByPath.has(entry.relativePath)) return false + if (entry.contentHash !== contentHash) return false + if (!areScannedAssetsInSync(entry.assets, scannedAssetHashes)) return false + return true + }) + if (candidates.length === 0) return null + if (candidates.length === 1) return candidates[0] + + let matchingBasename = candidates.filter(entry => { + return getFilename(entry.relativePath) === getFilename(file.relativePath) + }) + if (matchingBasename.length === 1) { + return matchingBasename[0] + } + + return null +} + +function getFilename(relativePath: string): string { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return relativePath + return parts[parts.length - 1] +} + +function getAssetFilesByIdFromManifest( + manifestEntry: ManifestEntry, +): Map { + let filesById = new Map() + for (let asset of manifestEntry.assets) { + if (!asset.id) continue + filesById.set(asset.id, asset.name) + } + return filesById +} + +function applyPathFromRelativePath( + content: string, + relativePath: string, + hasAssets: boolean, +): string { + let diskPath = derivePathFromRelativePath(relativePath, hasAssets) + let { frontmatter } = parseFrontmatter(content) + let currentPath = getPath(content) + + if (!frontmatter) { + if (!diskPath) return content + return `---\npath: ${diskPath}\n---\n\n${content}` + } + + if (currentPath === diskPath) return content + + if (currentPath && !diskPath) { + return content.replace( + /^(---\r?\n[\s\S]*?)path:\s*[^\r\n]*\r?\n([\s\S]*?---)/, + "$1$2", + ) + } + + if (currentPath && diskPath) { + return content.replace( + /^(---\r?\n[\s\S]*?)path:\s*[^\r\n]*/, + `$1path: ${diskPath}`, + ) + } + + if (!currentPath && diskPath) { + return content.replace(/^(---\r?\n)/, `$1path: ${diskPath}\n`) + } + + return content +} + +function derivePathFromRelativePath( + relativePath: string, + hasAssets: boolean, +): string | null { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length <= 1) return null + + let directoryParts = parts.slice(0, -1) + if (!hasAssets) { + let path = directoryParts.join("/") + return path || null + } + + let parentParts = directoryParts.slice(0, -1) + let path = parentParts.join("/") + return path || null +} + +function buildLocationFromRelativePath( + baseLocation: ReturnType extends Map< + string, + infer V + > + ? V + : never, + relativePath: string, +) { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return baseLocation + + return { + ...baseLocation, + dirPath: parts.slice(0, -1).join("/"), + filename: parts[parts.length - 1], + } +} + +async function hashBlob(blob: Blob): Promise { + let buffer = await blob.arrayBuffer() + let hashBuffer = await crypto.subtle.digest("SHA-256", buffer) + let hashArray = Array.from(new Uint8Array(hashBuffer)) + return hashArray + .map(b => b.toString(16).padStart(2, "0")) + .join("") + .slice(0, 16) +} + +async function fileExists( + dir: FileSystemDirectoryHandle, + name: string, +): Promise { + try { + await dir.getFileHandle(name) + return true + } catch { + return false + } +} + +async function assetsExistAtLocation( + dir: FileSystemDirectoryHandle, + loc: ReturnType extends Map + ? V + : never, + doc: BackupDoc, +): Promise { + let assetsDir = await dir.getDirectoryHandle("assets").catch(() => null) + if (!assetsDir) return false + + for (let asset of doc.assets) { + let filename = loc.assetFiles.get(asset.id) + if (!filename) return false + let exists = await fileExists(assetsDir, filename) + if (!exists) return false + } + + return true +} + +async function hashScannedAssets( + assets: { name: string; blob: Blob }[], +): Promise { + let hashed = await Promise.all( + assets.map(async asset => ({ + name: asset.name, + hash: await hashBlob(asset.blob), + })), + ) + return hashed +} + +function areScannedAssetsInSync( + manifestAssets: { id?: string; name: string; hash: string }[], + scannedAssets: ScannedAssetHash[], +): boolean { + if (manifestAssets.length !== scannedAssets.length) return false + + let sortedManifest = [...manifestAssets].sort((a, b) => + a.name.localeCompare(b.name), + ) + let sortedScanned = [...scannedAssets].sort((a, b) => + a.name.localeCompare(b.name), + ) + + for (let i = 0; i < sortedManifest.length; i++) { + if (sortedManifest[i].name !== sortedScanned[i].name) return false + if (sortedManifest[i].hash !== sortedScanned[i].hash) return false + } + + return true +} + +function areManifestAssetsEqual( + a: { id?: string; name: string; hash: string }[], + b: { id?: string; name: string; hash: string }[], +): boolean { + if (a.length !== b.length) return false + + let sortedA = [...a].sort((left, right) => + left.name.localeCompare(right.name), + ) + let sortedB = [...b].sort((left, right) => + left.name.localeCompare(right.name), + ) + + for (let i = 0; i < sortedA.length; i++) { + if ((sortedA[i].id ?? null) !== (sortedB[i].id ?? null)) return false + if (sortedA[i].name !== sortedB[i].name) return false + if (sortedA[i].hash !== sortedB[i].hash) return false + } + + return true +} + +function wasRecentlyImported(scopeKey: string, relativePath: string): boolean { + let key = makeRecentImportKey(scopeKey, relativePath) + let importedAt = recentImportedRelativePaths.get(key) + if (!importedAt) return false + if (Date.now() - importedAt > RECENT_IMPORT_WINDOW_MS) { + recentImportedRelativePaths.delete(key) + return false + } + return true +} + +function markRecentlyImported(scopeKey: string, relativePath: string): void { + recentImportedRelativePaths.set( + makeRecentImportKey(scopeKey, relativePath), + Date.now(), + ) +} + +function makeRecentImportKey(scopeKey: string, relativePath: string): string { + return `${scopeKey}:${relativePath}` +} + +function getDocumentListScopeKey(targetDocs: DocumentList): string { + let listId = targetDocs.$jazz.id + if (listId) return `docs:${listId}` + + return "docs:unknown" +} + +function getDocLocationKey(doc: BackupDoc): string { + let path = doc.path ?? "" + let hasAssets = doc.assets.length > 0 ? "assets" : "no-assets" + return `${doc.title}|${path}|${hasAssets}` +} + +function manifestEntryMatchesScope( + entry: ManifestEntry, + scopeId: string, +): boolean { + if (!entry.scopeId) return true + if (entry.scopeId === "docs:unknown") return true + return entry.scopeId === scopeId +} + +function buildManifestAssetsFromScanned( + scannedAssets: ScannedAssetHash[], + existingAssets: { id?: string; name: string; hash: string }[] = [], +): { id?: string; name: string; hash: string }[] { + let existingByName = new Map(existingAssets.map(asset => [asset.name, asset])) + return scannedAssets.map(asset => { + let existing = existingByName.get(asset.name) + if (!existing) { + return { name: asset.name, hash: asset.hash } + } + if (existing.hash !== asset.hash) { + return { name: asset.name, hash: asset.hash } + } + return { id: existing.id, name: asset.name, hash: asset.hash } + }) +} + +function upsertManifestEntry( + entriesByDocId: Map, + entry: ManifestEntry, +): boolean { + let existing = entriesByDocId.get(entry.docId) + if (existing && areManifestEntriesEquivalent(existing, entry)) { + return false + } + entriesByDocId.set(entry.docId, entry) + return true +} + +function haveManifestEntriesChanged( + left: ManifestEntry[], + right: ManifestEntry[], +): boolean { + if (left.length !== right.length) return true + + let sortedLeft = [...left].sort(compareManifestEntries) + let sortedRight = [...right].sort(compareManifestEntries) + for (let i = 0; i < sortedLeft.length; i++) { + if (!areManifestEntriesEquivalent(sortedLeft[i], sortedRight[i])) { + return true + } + } + + return false +} + +function compareManifestEntries( + left: ManifestEntry, + right: ManifestEntry, +): number { + if (left.docId !== right.docId) return left.docId.localeCompare(right.docId) + return left.relativePath.localeCompare(right.relativePath) +} + +function areManifestEntriesEquivalent( + left: ManifestEntry, + right: ManifestEntry, +): boolean { + if (left.docId !== right.docId) return false + if (left.relativePath !== right.relativePath) return false + if ((left.scopeId ?? null) !== (right.scopeId ?? null)) return false + if ((left.locationKey ?? null) !== (right.locationKey ?? null)) return false + if (left.contentHash !== right.contentHash) return false + if (left.lastSyncedAt !== right.lastSyncedAt) return false + if (!areManifestAssetsEqual(left.assets, right.assets)) return false + return true +} + +function findMatchingUntrackedDocId( + file: ScannedFile, + targetDocs: DocumentList, +): string | null { + if (file.assets.length > 0) return null + + let expectedContent = applyPathFromRelativePath( + file.content, + file.relativePath, + false, + ) + + for (let doc of targetDocs) { + if (!doc?.$isLoaded || doc.deletedAt) continue + if (!doc.content?.$isLoaded) continue + if (doc.assets?.$isLoaded && doc.assets.length > 0) continue + + if (doc.content.toString() === expectedContent) { + return doc.$jazz.id + } + } + + return null +} diff --git a/src/lib/backup-scenarios.test.ts b/src/lib/backup-scenarios.test.ts new file mode 100644 index 0000000..b0d37f0 --- /dev/null +++ b/src/lib/backup-scenarios.test.ts @@ -0,0 +1,800 @@ +import { beforeEach, describe, expect, it } from "vitest" +import { co, Group, FileStream } from "jazz-tools" +import { createJazzTestAccount, setupJazzTestSync } from "jazz-tools/testing" +import { UserAccount, Document, Asset, VideoAsset } from "@/schema" +import { getPath } from "@/editor/frontmatter" +import { getDocumentTitle } from "@/lib/document-utils" +import type { BackupDoc } from "./backup-sync" +import { readManifest } from "./backup-sync" +import { hashContent, syncBackup, syncFromBackup } from "./backup-engine" +import { + MockDirectoryHandle, + createMockBlob, + basename, + readFileAtPath as readFile, + removeFileAtPath as removeFile, + writeFileAtPath, +} from "./backup-test-helpers" + +type LoadedAccount = co.loaded +type LoadedDoc = co.loaded + +describe("backup scenarios", () => { + let account: LoadedAccount + let docs: co.loaded>> + let root: MockDirectoryHandle + + beforeEach(async () => { + await setupJazzTestSync() + account = await createJazzTestAccount({ + isCurrentActiveAccount: true, + AccountSchema: UserAccount, + }) + + let loaded = await account.$jazz.ensureLoaded({ + resolve: { + root: { + documents: { + $each: { + content: true, + assets: { $each: { image: true, video: true } }, + }, + }, + }, + }, + }) + + docs = loaded.root.documents + root = new MockDirectoryHandle("root") + }) + + it("new doc in alkalye", async () => { + let doc = await createDoc(docs, "# New Doc\n\nHello") + await pushToBackup(root, docs) + + let manifest = await readManifest(root) + expect(manifest?.entries.some(entry => entry.docId === doc.$jazz.id)).toBe( + true, + ) + expect(await hasFile(root, "New Doc.md")).toBe(true) + }) + + it("new doc locally", async () => { + let initialCount = getLoadedDocs(docs).length + root.addFile("Local Note.md", "# Local Note\n\nFrom filesystem", 2_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.created).toBe(1) + + let loadedDocs = getLoadedDocs(docs) + expect(loadedDocs).toHaveLength(initialCount + 1) + let imported = loadedDocs.find(d => getDocumentTitle(d) === "Local Note") + expect(imported).toBeDefined() + + let manifest = await readManifest(root) + expect( + manifest?.entries.some(entry => entry.relativePath === "Local Note.md"), + ).toBe(true) + }) + + it("does not duplicate imports on repeated pulls before manifest exists", async () => { + let initialCount = getLoadedDocs(docs).length + root.addFile("Repeated.md", "# Repeated\n\nFrom filesystem", 2_000) + + let first = await syncFromBackup(root, docs, true) + let second = await syncFromBackup(root, docs, true) + + expect(first.created).toBe(1) + expect(second.created).toBe(0) + expect(getLoadedDocs(docs)).toHaveLength(initialCount + 1) + }) + + it("does not create duplicates when same untracked no-asset doc already exists", async () => { + let initialCount = getLoadedDocs(docs).length + await createDoc(docs, "# Local Note\n\nFrom filesystem") + root.addFile("Local Note.md", "# Local Note\n\nFrom filesystem", 2_000) + + let result = await syncFromBackup(root, docs, true) + + expect(result.created).toBe(0) + expect(getLoadedDocs(docs)).toHaveLength(initialCount + 1) + }) + + it("imports same relative path across different backup roots", async () => { + let firstInitialCount = getLoadedDocs(docs).length + let firstRoot = new MockDirectoryHandle("first-root") + let secondRoot = new MockDirectoryHandle("second-root") + firstRoot.addFile( + "Cross Scope Note.md", + "# Cross Scope Note\n\nfrom first root", + 2_000, + ) + secondRoot.addFile( + "Cross Scope Note.md", + "# Cross Scope Note\n\nfrom second root", + 2_000, + ) + + let secondDocs = co.list(Document).create([], Group.create()) + let secondInitialCount = getLoadedDocs(secondDocs).length + + let firstResult = await syncFromBackup(firstRoot, docs, true) + let secondResult = await syncFromBackup(secondRoot, secondDocs, true) + + expect(firstResult.created).toBe(1) + expect(secondResult.created).toBe(1) + expect(getLoadedDocs(docs)).toHaveLength(firstInitialCount + 1) + expect(getLoadedDocs(secondDocs)).toHaveLength(secondInitialCount + 1) + }) + + it("imports asset references with created asset ids", async () => { + await writeFileAtPath( + root, + "Video Doc/Video Doc.md", + "# Video Doc\n\n![Clip](assets/clip.mp4)", + ) + await writeFileAtPath(root, "Video Doc/assets/clip.mp4", "video-bytes") + + let result = await syncFromBackup(root, docs, true) + expect(result.created).toBe(1) + + let imported = getLoadedDocs(docs).find( + d => getDocumentTitle(d) === "Video Doc", + ) + expect(imported).toBeDefined() + if (!imported?.content?.$isLoaded) throw new Error("Doc content not loaded") + + let content = imported.content.toString() + let refMatch = content.match(/!\[[^\]]*\]\(asset:([^)]+)\)/) + expect(refMatch).toBeTruthy() + + let refs = (content.match(/asset:[^)]+/g) ?? []).map(ref => + ref.replace("asset:", ""), + ) + let assetIds = (imported.assets?.$isLoaded ? [...imported.assets] : []) + .filter(asset => asset?.$isLoaded) + .map(asset => asset.$jazz.id) + + expect(assetIds.length).toBeGreaterThan(0) + for (let refId of refs) { + expect(assetIds).toContain(refId) + } + }) + + it("renamed in alkalye", async () => { + let doc = await createDoc(docs, "# Hello World") + await pushToBackup(root, docs) + expect(await hasFile(root, "Hello World.md")).toBe(true) + + if (!doc.content?.$isLoaded) throw new Error("Doc content not loaded") + doc.content.$jazz.applyDiff("# Another Title") + doc.$jazz.set("updatedAt", new Date(Date.now() + 2_000)) + await pushToBackup(root, docs) + + expect(await hasFile(root, "Another Title.md")).toBe(true) + expect(await hasFile(root, "Hello World.md")).toBe(false) + }) + + it("renamed locally", async () => { + let doc = await createDoc(docs, "# Hello") + await pushToBackup(root, docs) + let countBeforePull = getLoadedDocs(docs).length + + let oldFile = await readFile(root, "Hello.md") + await removeFile(root, "Hello.md") + root.addFile("Renamed Locally.md", oldFile, 5_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.created).toBe(0) + + let loadedDocs = getLoadedDocs(docs) + expect(loadedDocs).toHaveLength(countBeforePull) + expect(loadedDocs.some(d => d.$jazz.id === doc.$jazz.id)).toBe(true) + }) + + it("renamed folder locally updates path in alkalye", async () => { + await createDoc(docs, "---\npath: work\n---\n\n# Folder Move") + await pushToBackup(root, docs) + + let source = await readFile(root, "work/Folder Move.md") + await removeFile(root, "work/Folder Move.md") + root.addFile("archive/Folder Move.md", source, 8_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find( + d => getDocumentTitle(d) === "Folder Move", + ) + expect(loaded).toBeDefined() + expect(getPath(loaded?.content?.toString() ?? "")).toBe("archive") + }) + + it("imports file updates when lastModified equals last pull timestamp", async () => { + let doc = await createDoc(docs, "# Timestamp Edge\n\noriginal") + await pushToBackup(root, docs) + + let updatedContent = "# Timestamp Edge\n\nchanged on disk" + root.addFile("Timestamp Edge.md", updatedContent, 10_000) + + let result = await syncFromBackup(root, docs, true, 10_000) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + expect(loaded?.content?.toString()).toContain("changed on disk") + }) + + it("matches moved doc by filename when content hashes collide", async () => { + let first = await createDoc(docs, "# Same") + let second = await createDoc(docs, "# Same") + await pushToBackup(root, docs) + + let manifest = await readManifest(root) + if (!manifest) throw new Error("Manifest not found") + + let firstEntry = manifest.entries.find( + entry => entry.docId === first.$jazz.id, + ) + let secondEntry = manifest.entries.find( + entry => entry.docId === second.$jazz.id, + ) + if (!firstEntry || !secondEntry) throw new Error("Manifest entries missing") + + let secondContent = await readFile(root, secondEntry.relativePath) + await removeFile(root, firstEntry.relativePath) + await removeFile(root, secondEntry.relativePath) + + let movedFilename = basename(secondEntry.relativePath) + await writeFileAtPath(root, `archive/${movedFilename}`, secondContent) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBe(1) + expect(result.deleted).toBe(1) + + let firstLoaded = getLoadedDocs(docs).find( + d => d.$jazz.id === first.$jazz.id, + ) + let secondLoaded = getLoadedDocs(docs).find( + d => d.$jazz.id === second.$jazz.id, + ) + + expect(firstLoaded?.deletedAt).toBeTruthy() + expect(secondLoaded?.deletedAt).toBeFalsy() + }) + + it("matches moved docs with colliding hashes by asset hashes", async () => { + let first = await createDoc(docs, "# Same") + let second = await createDoc(docs, "# Same") + + let backupDocs: BackupDoc[] = [ + { + id: first.$jazz.id, + title: "Same", + content: "# Same\n\n![Clip](asset:asset-1)", + path: "alpha", + updatedAtMs: Date.now(), + assets: [ + { + id: "asset-1", + name: "clip", + blob: createMockBlob("first-video", "video/mp4"), + }, + ], + }, + { + id: second.$jazz.id, + title: "Same", + content: "# Same\n\n![Clip](asset:asset-2)", + path: "beta", + updatedAtMs: Date.now(), + assets: [ + { + id: "asset-2", + name: "clip", + blob: createMockBlob("second-video", "video/mp4"), + }, + ], + }, + ] + + await syncBackup(root, backupDocs) + + let firstOriginalPath = "alpha/Same/Same.md" + let secondOriginalPath = "beta/Same/Same.md" + let firstContent = await readFile(root, firstOriginalPath) + let secondContent = await readFile(root, secondOriginalPath) + await removeFile(root, firstOriginalPath) + await removeFile(root, secondOriginalPath) + + await writeFileAtPath(root, "archive/one/Same/Same.md", firstContent) + await writeFileAtPath( + root, + "archive/one/Same/assets/clip.mp4", + "first-video", + ) + await writeFileAtPath(root, "archive/two/Same/Same.md", secondContent) + await writeFileAtPath( + root, + "archive/two/Same/assets/clip.mp4", + "second-video", + ) + + let beforePullCount = getLoadedDocs(docs).length + let result = await syncFromBackup(root, docs, true) + + expect(result.created).toBe(0) + expect(result.updated).toBe(2) + expect(result.deleted).toBe(0) + expect(getLoadedDocs(docs)).toHaveLength(beforePullCount) + + let firstLoaded = getLoadedDocs(docs).find( + d => d.$jazz.id === first.$jazz.id, + ) + let secondLoaded = getLoadedDocs(docs).find( + d => d.$jazz.id === second.$jazz.id, + ) + expect(firstLoaded?.deletedAt).toBeFalsy() + expect(secondLoaded?.deletedAt).toBeFalsy() + expect(getPath(firstLoaded?.content?.toString() ?? "")).toBe("archive/one") + expect(getPath(secondLoaded?.content?.toString() ?? "")).toBe("archive/two") + }) + + it("keeps asset refs stable when pulling updates for docs with assets", async () => { + let { doc, assetId } = await createDocWithVideoAsset(docs, "Asset Sync") + let localContent = + "# Asset Sync\n\n![Clip](assets/clip.mp4)\n\nExternal edit" + let localPath = "Asset Sync.md" + await writeFileAtPath(root, localPath, localContent) + let contentHash = await hashContent( + "# Asset Sync\n\n![Clip](assets/clip.mp4)", + ) + let manifestContent = JSON.stringify( + { + version: 1, + entries: [ + { + docId: doc.$jazz.id, + relativePath: localPath, + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: [{ id: assetId, name: "clip.mp4", hash: "asset-hash" }], + }, + ], + lastSyncAt: new Date().toISOString(), + }, + null, + 2, + ) + await writeFileAtPath(root, ".alkalye-manifest.json", manifestContent) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + if (!loaded?.content?.$isLoaded) throw new Error("Doc content not loaded") + + let content = loaded.content.toString() + expect(content).toContain(`asset:${assetId}`) + expect(content).not.toContain("(assets/") + }) + + it("imports asset binary updates when markdown is unchanged", async () => { + let { doc, assetId } = await createDocWithVideoAsset(docs, "Binary Update") + let originalAsset = doc.assets?.$isLoaded + ? [...doc.assets].find( + asset => asset?.$isLoaded && asset.$jazz.id === assetId, + ) + : undefined + let originalVideoId = + originalAsset?.$isLoaded && + originalAsset.type === "video" && + originalAsset.video?.$isLoaded + ? originalAsset.video.$jazz.id + : null + let localPath = "Binary Update/Binary Update.md" + let localContent = "# Binary Update\n\n![Clip](assets/clip.mp4)" + await writeFileAtPath(root, localPath, localContent) + await writeFileAtPath( + root, + "Binary Update/assets/clip.mp4", + "new-video-bytes", + ) + + let contentHash = await hashContent(localContent) + let previousAssetHash = await hashContent("old-video-bytes") + await writeFileAtPath( + root, + ".alkalye-manifest.json", + JSON.stringify( + { + version: 1, + entries: [ + { + docId: doc.$jazz.id, + relativePath: localPath, + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: [ + { + id: assetId, + name: "clip.mp4", + hash: previousAssetHash, + }, + ], + }, + ], + lastSyncAt: new Date().toISOString(), + }, + null, + 2, + ), + ) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + let updatedAsset = loaded?.assets?.$isLoaded + ? [...loaded.assets].find( + asset => asset?.$isLoaded && asset.$jazz.id === assetId, + ) + : undefined + expect(updatedAsset?.$isLoaded).toBe(true) + if ( + !updatedAsset?.$isLoaded || + updatedAsset.type !== "video" || + !updatedAsset.video?.$isLoaded + ) { + throw new Error("Updated video asset not loaded") + } + + expect(updatedAsset.video.$jazz.id).not.toBe(originalVideoId) + }) + + it("imports asset-only updates when markdown file is older than last pull", async () => { + let { doc, assetId } = await createDocWithVideoAsset( + docs, + "Asset Timestamp", + ) + let originalAsset = doc.assets?.$isLoaded + ? [...doc.assets].find( + asset => asset?.$isLoaded && asset.$jazz.id === assetId, + ) + : undefined + let originalVideoId = + originalAsset?.$isLoaded && + originalAsset.type === "video" && + originalAsset.video?.$isLoaded + ? originalAsset.video.$jazz.id + : null + + let docDir = new MockDirectoryHandle("Asset Timestamp") + let assetsDir = new MockDirectoryHandle("assets") + docDir.addDirectory("assets", assetsDir) + docDir.addFile( + "Asset Timestamp.md", + "# Asset Timestamp\n\n![Clip](assets/clip.mp4)", + 1_000, + ) + assetsDir.addFile("clip.mp4", "new-video-bytes", 9_000) + root.addDirectory("Asset Timestamp", docDir) + + let contentHash = await hashContent( + "# Asset Timestamp\n\n![Clip](assets/clip.mp4)", + ) + let previousAssetHash = await hashContent("old-video-bytes") + await writeFileAtPath( + root, + ".alkalye-manifest.json", + JSON.stringify( + { + version: 1, + entries: [ + { + docId: doc.$jazz.id, + relativePath: "Asset Timestamp/Asset Timestamp.md", + contentHash, + lastSyncedAt: new Date().toISOString(), + assets: [ + { + id: assetId, + name: "clip.mp4", + hash: previousAssetHash, + }, + ], + }, + ], + lastSyncAt: new Date().toISOString(), + }, + null, + 2, + ), + ) + + let result = await syncFromBackup(root, docs, true, 5_000) + expect(result.updated).toBe(1) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + let updatedAsset = loaded?.assets?.$isLoaded + ? [...loaded.assets].find( + asset => asset?.$isLoaded && asset.$jazz.id === assetId, + ) + : undefined + expect(updatedAsset?.$isLoaded).toBe(true) + if ( + !updatedAsset?.$isLoaded || + updatedAsset.type !== "video" || + !updatedAsset.video?.$isLoaded + ) { + throw new Error("Updated video asset not loaded") + } + + expect(updatedAsset.video.$jazz.id).not.toBe(originalVideoId) + }) + + it("writes asset ids to manifest entries on backup", async () => { + let backupDocs: BackupDoc[] = [ + { + id: "doc-asset-manifest", + title: "Asset Manifest", + content: "# Asset Manifest\n\n![Clip](asset:asset-1)", + path: null, + updatedAtMs: Date.now(), + assets: [ + { + id: "asset-1", + name: "clip", + blob: createMockBlob("video", "video/mp4"), + }, + ], + }, + ] + + await syncBackup(root, backupDocs) + + let manifest = await readManifest(root) + expect(manifest).toBeDefined() + expect(manifest?.entries[0].assets[0].id).toBe("asset-1") + }) + + it("removes stale files from assets folders during backup", async () => { + let first: BackupDoc[] = [ + { + id: "doc-assets-prune", + title: "Assets Prune", + content: + "# Assets Prune\n\n![One](asset:asset-1)\n![Two](asset:asset-2)", + path: null, + updatedAtMs: Date.now(), + assets: [ + { + id: "asset-1", + name: "one", + blob: createMockBlob("one", "image/png"), + }, + { + id: "asset-2", + name: "two", + blob: createMockBlob("two", "image/png"), + }, + ], + }, + ] + await syncBackup(root, first) + expect(await hasFile(root, "Assets Prune/assets/two.png")).toBe(true) + + let second: BackupDoc[] = [ + { + id: "doc-assets-prune", + title: "Assets Prune", + content: "# Assets Prune\n\n![One](asset:asset-1)", + path: null, + updatedAtMs: Date.now() + 1, + assets: [ + { + id: "asset-1", + name: "one", + blob: createMockBlob("one", "image/png"), + }, + ], + }, + ] + await syncBackup(root, second) + + expect(await hasFile(root, "Assets Prune/assets/one.png")).toBe(true) + expect(await hasFile(root, "Assets Prune/assets/two.png")).toBe(false) + }) + + it("changed path in alkalye", async () => { + let doc = await createDoc(docs, "# Path Doc") + await pushToBackup(root, docs) + + if (!doc.content?.$isLoaded) throw new Error("Doc content not loaded") + doc.content.$jazz.applyDiff("---\npath: work/notes\n---\n\n# Path Doc") + doc.$jazz.set("updatedAt", new Date(Date.now() + 3_000)) + await pushToBackup(root, docs) + + expect(await hasFile(root, "work/notes/Path Doc.md")).toBe(true) + expect(await hasFile(root, "Path Doc.md")).toBe(false) + }) + + it("changed path locally is normalized to filesystem location", async () => { + await createDoc(docs, "---\npath: work\n---\n\n# Local Path") + await pushToBackup(root, docs) + + let source = "---\npath: notes\n---\n\n# Local Path" + root.addFile("work/Local Path.md", source, 7_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.updated).toBeGreaterThanOrEqual(0) + + let loaded = getLoadedDocs(docs).find( + d => getDocumentTitle(d) === "Local Path", + ) + expect(loaded).toBeDefined() + expect(getPath(loaded?.content?.toString() ?? "")).toBe("work") + }) + + it("deleted in alkalye", async () => { + let doc = await createDoc(docs, "# Delete Me") + await pushToBackup(root, docs) + expect(await hasFile(root, "Delete Me.md")).toBe(true) + + doc.$jazz.set("deletedAt", new Date()) + doc.$jazz.set("updatedAt", new Date(Date.now() + 4_000)) + await pushToBackup(root, docs) + + expect(await hasFile(root, "Delete Me.md")).toBe(false) + let manifest = await readManifest(root) + expect(manifest?.entries.some(entry => entry.docId === doc.$jazz.id)).toBe( + false, + ) + }) + + it("deleted locally", async () => { + let doc = await createDoc(docs, "# Remove Local") + await pushToBackup(root, docs) + await removeFile(root, "Remove Local.md") + + let result = await syncFromBackup(root, docs, true) + expect(result.deleted).toBe(1) + + let target = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(target?.deletedAt).toBeTruthy() + }) + + it("re-writes missing backup file even when manifest says synced", async () => { + await createDoc(docs, "# Rewrite Missing") + await pushToBackup(root, docs) + await removeFile(root, "Rewrite Missing.md") + + await pushToBackup(root, docs) + + expect(await hasFile(root, "Rewrite Missing.md")).toBe(true) + }) + + it("edited both locally and in alkalye keeps document accessible and stable", async () => { + let doc = await createDoc(docs, "# Conflict\n\nbase") + await pushToBackup(root, docs) + + if (!doc.content?.$isLoaded) throw new Error("Doc content not loaded") + doc.content.$jazz.applyDiff("# Conflict\n\nbase\nfrom-alkalye") + doc.$jazz.set("updatedAt", new Date(Date.now() + 5_000)) + + let existing = await readFile(root, "Conflict.md") + root.addFile("Conflict.md", `${existing}\nfrom-local`, 9_000) + + let result = await syncFromBackup(root, docs, true) + expect(result.errors).toHaveLength(0) + + let loaded = getLoadedDocs(docs).find(d => d.$jazz.id === doc.$jazz.id) + expect(loaded).toBeDefined() + expect(loaded?.deletedAt).toBeFalsy() + expect(loaded?.content?.toString()).toContain("Conflict") + }) +}) + +function getLoadedDocs( + docs: co.loaded>>, +): LoadedDoc[] { + let result: LoadedDoc[] = [] + for (let doc of docs) { + if (doc?.$isLoaded) result.push(doc) + } + return result +} + +async function createDoc( + docs: co.loaded>>, + content: string, +): Promise { + let group = Group.create() + let now = new Date() + let doc = Document.create( + { + version: 1, + content: co.plainText().create(content, group), + createdAt: now, + updatedAt: now, + }, + group, + ) + docs.$jazz.push(doc) + return doc +} + +async function createDocWithVideoAsset( + docs: co.loaded>>, + title: string, +): Promise<{ doc: LoadedDoc; assetId: string }> { + let group = Group.create() + let now = new Date() + let stream = await FileStream.createFromBlob( + createMockBlob("video", "video/mp4"), + { + owner: group, + }, + ) + let videoAsset = VideoAsset.create( + { + type: "video", + name: "clip", + video: stream, + mimeType: "video/mp4", + createdAt: now, + }, + group, + ) + let content = `# ${title}\n\n![Clip](asset:${videoAsset.$jazz.id})` + let doc = Document.create( + { + version: 1, + content: co.plainText().create(content, group), + assets: co.list(Asset).create([videoAsset], group), + createdAt: now, + updatedAt: now, + }, + group, + ) + docs.$jazz.push(doc) + if (!doc.$isLoaded) throw new Error("Doc failed to load") + return { doc, assetId: videoAsset.$jazz.id } +} + +async function pushToBackup( + handle: FileSystemDirectoryHandle, + docs: co.loaded>>, +): Promise { + let backupDocs: BackupDoc[] = [] + + for (let doc of docs) { + if (!doc?.$isLoaded || doc.deletedAt) continue + let content = doc.content?.toString() ?? "" + backupDocs.push({ + id: doc.$jazz.id, + title: getDocumentTitle(doc), + content, + path: getPath(content), + updatedAtMs: doc.updatedAt?.getTime() ?? 0, + assets: [], + }) + } + + await syncBackup(handle, backupDocs) +} + +async function hasFile( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + try { + await readFile(root, relativePath) + return true + } catch { + return false + } +} diff --git a/src/lib/backup-settings-ui.tsx b/src/lib/backup-settings-ui.tsx new file mode 100644 index 0000000..746811f --- /dev/null +++ b/src/lib/backup-settings-ui.tsx @@ -0,0 +1,374 @@ +import { useState } from "react" +import { FolderOpen, AlertCircle } from "lucide-react" +import { Button } from "@/components/ui/button" +import { Switch } from "@/components/ui/switch" +import { Label } from "@/components/ui/label" +import { + useBackupStore, + enableBackup, + disableBackup, + changeBackupDirectory, + useSpaceBackupPath, + setSpaceBackupHandle, + clearSpaceBackupHandle, + supportsFileSystemWatch, + isBackupSupported, +} from "@/lib/backup-storage" + +export { BackupSettings, SpaceBackupSettings } + +function BackupSettings() { + let { + enabled, + bidirectional, + directoryName, + lastBackupAt, + lastPullAt, + lastError, + setBidirectional, + setLastError, + } = useBackupStore() + let [pendingAction, setPendingAction] = useState< + "enable" | "disable" | "change" | null + >(null) + let isLoading = pendingAction !== null + let canWatchFileSystem = supportsFileSystemWatch() + + if (!isBackupSupported()) { + return + } + + async function handleEnable() { + setPendingAction("enable") + setLastError(null) + try { + let result = await enableBackup() + if (!result.success && result.error && result.error !== "Cancelled") { + setLastError(result.error) + } + } finally { + setPendingAction(null) + } + } + + async function handleDisable() { + setPendingAction("disable") + try { + await disableBackup() + } finally { + setPendingAction(null) + } + } + + async function handleChangeDirectory() { + setPendingAction("change") + setLastError(null) + try { + let result = await changeBackupDirectory() + if (!result.success && result.error && result.error !== "Cancelled") { + setLastError(result.error) + } + } finally { + setPendingAction(null) + } + } + + let lastBackupDate = lastBackupAt ? new Date(lastBackupAt) : null + let formattedLastBackup = lastBackupDate + ? lastBackupDate.toLocaleString() + : null + + let lastPullDate = lastPullAt ? new Date(lastPullAt) : null + let formattedLastPull = lastPullDate ? lastPullDate.toLocaleString() : null + + return ( +
+

+ Local Backup +

+
+ {enabled ? ( + <> +
+ + + {bidirectional ? "Syncing" : "Backing up"} to folder + +
+

+ Folder:{" "} + + {directoryName} + +

+ {formattedLastBackup && ( +

+ Last backup: {formattedLastBackup} +

+ )} + {bidirectional && formattedLastPull && ( +

+ Last sync: {formattedLastPull} +

+ )} + {lastError && ( +
+ + {lastError} +
+ )} +
+
+
+ +

+ {canWatchFileSystem + ? "Import folder edits back into Alkalye automatically." + : "Requires Chromium with File System Observer support."} +

+
+ +
+
+
+ + +
+ + ) : ( + <> +
+ Automatic backup disabled +
+

+ Automatically back up your documents to a folder on this device. +

+ + + )} +
+
+ ) +} + +interface SpaceBackupSettingsProps { + spaceId: string + isAdmin: boolean +} + +function SpaceBackupSettings({ spaceId, isAdmin }: SpaceBackupSettingsProps) { + let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) + let [pendingAction, setPendingAction] = useState< + "choose" | "change" | "clear" | null + >(null) + let [error, setError] = useState(null) + let isLoading = pendingAction !== null + + if (!isBackupSupported()) { + return + } + + async function handleChooseFolder() { + setPendingAction("choose") + setError(null) + try { + let handle = await window.showDirectoryPicker({ mode: "readwrite" }) + await setSpaceBackupHandle(spaceId, handle) + setDirectoryName(handle.name) + } catch (e) { + if (!(e instanceof Error && e.name === "AbortError")) { + setError("Failed to choose folder. Try again.") + console.error("Failed to select folder:", e) + } + } finally { + setPendingAction(null) + } + } + + async function handleChangeFolder() { + setPendingAction("change") + setError(null) + try { + let handle = await window.showDirectoryPicker({ mode: "readwrite" }) + await setSpaceBackupHandle(spaceId, handle) + setDirectoryName(handle.name) + } catch (e) { + if (!(e instanceof Error && e.name === "AbortError")) { + setError("Failed to choose folder. Try again.") + console.error("Failed to select folder:", e) + } + } finally { + setPendingAction(null) + } + } + + async function handleClear() { + setPendingAction("clear") + setError(null) + try { + await clearSpaceBackupHandle(spaceId) + setDirectoryName(null) + } catch { + setError("Failed to clear folder. Try again.") + } finally { + setPendingAction(null) + } + } + + return ( +
+

+ Local Backup +

+
+ {directoryName ? ( + <> +
+ + Backup folder set +
+

+ Folder:{" "} + + {directoryName} + +

+ {error && ( +
+ + {error} +
+ )} +
+ + +
+ {!isAdmin && ( +

+ Only space admins can change this folder. +

+ )} + + ) : ( + <> +
+ No backup folder set +
+

+ Set a backup folder for this space's documents. +

+ {error && ( +
+ + {error} +
+ )} + + {!isAdmin && ( +

+ Only space admins can set a backup folder. +

+ )} + + )} +
+
+ ) +} + +function UnsupportedBrowserCallout() { + return ( +
+

+ Local Backup +

+
+
+ +
+

+ Local backup requires a Chromium-based browser (Chrome, Edge, + Brave, or Opera). +

+

+ Safari and Firefox do not support the File System Access API + needed for this feature. +

+
+
+
+
+ ) +} diff --git a/src/lib/backup-storage.ts b/src/lib/backup-storage.ts new file mode 100644 index 0000000..0a1a3cb --- /dev/null +++ b/src/lib/backup-storage.ts @@ -0,0 +1,304 @@ +import { useState } from "react" +import { create } from "zustand" +import { persist } from "zustand/middleware" +import { get as idbGet, set as idbSet, del as idbDel } from "idb-keyval" + +export { + BACKUP_DEBOUNCE_MS, + SPACE_BACKUP_DEBOUNCE_MS, + SPACE_BACKUP_KEY_PREFIX, + useBackupStore, + enableBackup, + disableBackup, + changeBackupDirectory, + checkBackupPermission, + getSpaceBackupPath, + setSpaceBackupPath, + clearSpaceBackupPath, + useSpaceBackupPath, + getBackupHandle, + getSpaceBackupHandle, + setSpaceBackupHandle, + clearSpaceBackupHandle, + supportsFileSystemWatch, + isBackupSupported, + observeDirectoryChanges, + toTimestamp, +} + +declare global { + interface FileSystemObserver { + observe( + handle: FileSystemDirectoryHandle, + options?: { recursive?: boolean }, + ): Promise + disconnect(): void + } + + interface Window { + FileSystemObserver?: { + new ( + onChange: (records: unknown[], observer: FileSystemObserver) => void, + ): FileSystemObserver + } + showDirectoryPicker(options?: { + mode?: "read" | "readwrite" + }): Promise + } + + interface FileSystemDirectoryHandle { + entries(): AsyncIterableIterator<[string, FileSystemHandle]> + queryPermission(options: { + mode: "read" | "readwrite" + }): Promise<"granted" | "denied" | "prompt"> + requestPermission(options: { + mode: "read" | "readwrite" + }): Promise<"granted" | "denied" | "prompt"> + } +} + +let BACKUP_DEBOUNCE_MS = 1200 +let SPACE_BACKUP_DEBOUNCE_MS = 1200 +let HANDLE_STORAGE_KEY = "backup-directory-handle" +let SPACE_BACKUP_KEY_PREFIX = "backup-settings-space-" + +interface BackupState { + enabled: boolean + bidirectional: boolean + directoryName: string | null + lastBackupAt: string | null + lastPullAt: string | null + lastError: string | null + setEnabled: (enabled: boolean) => void + setBidirectional: (bidirectional: boolean) => void + setDirectoryName: (name: string | null) => void + setLastBackupAt: (date: string | null) => void + setLastPullAt: (date: string | null) => void + setLastError: (error: string | null) => void + reset: () => void +} + +interface SpaceBackupState { + directoryName: string | null +} + +let useBackupStore = create()( + persist( + set => ({ + enabled: false, + bidirectional: true, + directoryName: null, + lastBackupAt: null, + lastPullAt: null, + lastError: null, + setEnabled: enabled => set({ enabled }), + setBidirectional: bidirectional => set({ bidirectional }), + setDirectoryName: directoryName => set({ directoryName }), + setLastBackupAt: lastBackupAt => set({ lastBackupAt }), + setLastPullAt: lastPullAt => set({ lastPullAt }), + setLastError: lastError => set({ lastError }), + reset: () => + set({ + enabled: false, + bidirectional: true, + directoryName: null, + lastBackupAt: null, + lastPullAt: null, + lastError: null, + }), + }), + { name: "backup-settings" }, + ), +) + +async function enableBackup(): Promise<{ + success: boolean + directoryName?: string + error?: string +}> { + let handle = await requestBackupDirectory() + if (!handle) return { success: false, error: "Cancelled" } + + useBackupStore.getState().setEnabled(true) + useBackupStore.getState().setDirectoryName(handle.name) + useBackupStore.getState().setLastError(null) + + return { success: true, directoryName: handle.name } +} + +async function disableBackup(): Promise { + await clearHandle() + useBackupStore.getState().reset() +} + +async function changeBackupDirectory(): Promise<{ + success: boolean + directoryName?: string + error?: string +}> { + let handle = await requestBackupDirectory() + if (!handle) return { success: false, error: "Cancelled" } + + useBackupStore.getState().setDirectoryName(handle.name) + useBackupStore.getState().setLastError(null) + return { success: true, directoryName: handle.name } +} + +async function checkBackupPermission(): Promise { + let handle = await getBackupHandle() + return handle !== null +} + +function getSpaceBackupPath(spaceId: string): string | null { + try { + let key = getSpaceBackupStorageKey(spaceId) + let stored = localStorage.getItem(key) + if (!stored) return null + let parsed = JSON.parse(stored) + if (!isSpaceBackupState(parsed)) return null + return parsed.directoryName + } catch { + return null + } +} + +function setSpaceBackupPath(spaceId: string, directoryName: string): void { + let key = getSpaceBackupStorageKey(spaceId) + let state: SpaceBackupState = { directoryName } + localStorage.setItem(key, JSON.stringify(state)) +} + +function clearSpaceBackupPath(spaceId: string): void { + let key = getSpaceBackupStorageKey(spaceId) + localStorage.removeItem(key) +} + +function useSpaceBackupPath(spaceId: string): { + directoryName: string | null + setDirectoryName: (name: string | null) => void +} { + let [directoryName, setDirectoryNameState] = useState(() => + getSpaceBackupPath(spaceId), + ) + + function setDirectoryName(name: string | null) { + if (name) { + setSpaceBackupPath(spaceId, name) + } else { + clearSpaceBackupPath(spaceId) + } + setDirectoryNameState(name) + } + + return { directoryName, setDirectoryName } +} + +async function getBackupHandle(): Promise { + let handle = await getStoredHandle() + if (!handle) return null + let hasPermission = await verifyPermission(handle) + if (!hasPermission) return null + return handle +} + +async function getSpaceBackupHandle( + spaceId: string, +): Promise { + try { + let handle = await idbGet( + `${HANDLE_STORAGE_KEY}-space-${spaceId}`, + ) + if (!handle) return null + let hasPermission = await verifyPermission(handle) + if (!hasPermission) return null + return handle + } catch { + return null + } +} + +async function setSpaceBackupHandle( + spaceId: string, + handle: FileSystemDirectoryHandle, +): Promise { + await idbSet(`${HANDLE_STORAGE_KEY}-space-${spaceId}`, handle) +} + +async function clearSpaceBackupHandle(spaceId: string): Promise { + await idbDel(`${HANDLE_STORAGE_KEY}-space-${spaceId}`) +} + +function supportsFileSystemWatch(): boolean { + return typeof window.FileSystemObserver === "function" +} + +function isBackupSupported(): boolean { + return "showDirectoryPicker" in window +} + +async function observeDirectoryChanges( + handle: FileSystemDirectoryHandle, + onChange: () => void, +): Promise<(() => void) | null> { + let Observer = window.FileSystemObserver + if (!Observer) return null + + let observer = new Observer(() => { + onChange() + }) + await observer.observe(handle, { recursive: true }) + return () => observer.disconnect() +} + +function toTimestamp(value: string | null): number | null { + if (!value) return null + let ms = Date.parse(value) + return Number.isNaN(ms) ? null : ms +} + +function getSpaceBackupStorageKey(spaceId: string): string { + return `${SPACE_BACKUP_KEY_PREFIX}${spaceId}` +} + +function isSpaceBackupState(value: unknown): value is SpaceBackupState { + if (typeof value !== "object" || value === null) return false + if (!("directoryName" in value)) return false + return value.directoryName === null || typeof value.directoryName === "string" +} + +async function getStoredHandle(): Promise { + try { + let handle = await idbGet(HANDLE_STORAGE_KEY) + return handle ?? null + } catch { + return null + } +} + +async function storeHandle(handle: FileSystemDirectoryHandle): Promise { + await idbSet(HANDLE_STORAGE_KEY, handle) +} + +async function clearHandle(): Promise { + await idbDel(HANDLE_STORAGE_KEY) +} + +async function verifyPermission( + handle: FileSystemDirectoryHandle, +): Promise { + let opts: { mode: "readwrite" } = { mode: "readwrite" } + if ((await handle.queryPermission(opts)) === "granted") return true + if ((await handle.requestPermission(opts)) === "granted") return true + return false +} + +async function requestBackupDirectory(): Promise { + try { + let handle = await window.showDirectoryPicker({ mode: "readwrite" }) + await storeHandle(handle) + return handle + } catch (e) { + if (e instanceof Error && e.name === "AbortError") return null + throw e + } +} diff --git a/src/lib/backup-subscribers.test.ts b/src/lib/backup-subscribers.test.ts new file mode 100644 index 0000000..30897b9 --- /dev/null +++ b/src/lib/backup-subscribers.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest" +import { spaceBackupDocumentResolve } from "./backup-subscribers" + +describe("space backup subscriber query", () => { + it("resolves image and video assets", () => { + expect(spaceBackupDocumentResolve.documents.$each.assets.$each.image).toBe( + true, + ) + expect(spaceBackupDocumentResolve.documents.$each.assets.$each.video).toBe( + true, + ) + }) +}) diff --git a/src/lib/backup-subscribers.tsx b/src/lib/backup-subscribers.tsx new file mode 100644 index 0000000..29793d3 --- /dev/null +++ b/src/lib/backup-subscribers.tsx @@ -0,0 +1,362 @@ +import { useEffect, useRef, useState } from "react" +import { useAccount, useCoState } from "jazz-tools/react" +import { type ResolveQuery, Group } from "jazz-tools" +import { UserAccount, Space } from "@/schema" +import { + syncBackup, + syncFromBackup, + prepareBackupDoc, + type LoadedDocument, +} from "@/lib/backup-engine" +import { + BACKUP_DEBOUNCE_MS, + SPACE_BACKUP_DEBOUNCE_MS, + SPACE_BACKUP_KEY_PREFIX, + useBackupStore, + useSpaceBackupPath, + getSpaceBackupPath, + getBackupHandle, + getSpaceBackupHandle, + supportsFileSystemWatch, + observeDirectoryChanges, + toTimestamp, +} from "@/lib/backup-storage" + +export { BackupSubscriber, SpacesBackupSubscriber, spaceBackupDocumentResolve } + +let spaceBackupDocumentResolve = { + documents: { + $each: { + content: true, + assets: { $each: { image: true, video: true } }, + }, + $onError: "catch", + }, +} as const + +let backupQuery = { + root: { + documents: { + $each: { content: true, assets: { $each: { image: true, video: true } } }, + $onError: "catch", + }, + }, +} as const satisfies ResolveQuery + +let spacesBackupQuery = { + root: { + spaces: true, + }, +} as const satisfies ResolveQuery + +let spaceLastPullAtById = new Map() + +function BackupSubscriber() { + let { + enabled, + bidirectional, + lastPullAt, + setLastBackupAt, + setLastPullAt, + setLastError, + setEnabled, + setDirectoryName, + } = useBackupStore() + let me = useAccount(UserAccount, { resolve: backupQuery }) + let debounceRef = useRef | null>(null) + let lastContentHashRef = useRef("") + let lastPullAtRef = useRef(toTimestamp(lastPullAt)) + let isPushingRef = useRef(false) + let isPullingRef = useRef(false) + + useEffect(() => { + lastPullAtRef.current = toTimestamp(lastPullAt) + }, [lastPullAt]) + + useEffect(() => { + if (!enabled || !me.$isLoaded) return + + let docs = me.root?.documents + if (!docs?.$isLoaded) return + + let activeDocs = [...docs].filter(d => d?.$isLoaded && !d.deletedAt) + let contentHash = activeDocs + .map(d => `${d.$jazz.id}:${d.updatedAt?.getTime()}`) + .sort() + .join("|") + + if (contentHash === lastContentHashRef.current) return + lastContentHashRef.current = contentHash + + if (debounceRef.current) clearTimeout(debounceRef.current) + debounceRef.current = setTimeout(async () => { + try { + let handle = await getBackupHandle() + if (!handle) { + setEnabled(false) + setDirectoryName(null) + setLastError("Permission lost - please re-enable backup") + return + } + + let loadedDocs = activeDocs.filter( + (d): d is LoadedDocument => d?.$isLoaded === true, + ) + let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) + let scopeId = docs.$jazz.id ? `docs:${docs.$jazz.id}` : "docs:unknown" + isPushingRef.current = true + await syncBackup(handle, backupDocs, scopeId) + + setLastBackupAt(new Date().toISOString()) + setLastError(null) + } catch (e) { + setLastError(e instanceof Error ? e.message : "Backup failed") + } finally { + isPushingRef.current = false + } + }, BACKUP_DEBOUNCE_MS) + + return () => { + if (debounceRef.current) clearTimeout(debounceRef.current) + } + }, [enabled, me, setLastBackupAt, setLastError, setEnabled, setDirectoryName]) + + useEffect(() => { + if (!enabled || !bidirectional || !me.$isLoaded) return + if (!supportsFileSystemWatch()) return + + let docs = me.root?.documents + if (!docs?.$isLoaded) return + + async function doPull() { + try { + if (isPushingRef.current) return + if (isPullingRef.current) return + isPullingRef.current = true + let handle = await getBackupHandle() + if (!handle) return + + if (!docs.$isLoaded || !isDocumentList(docs)) return + let result = await syncFromBackup( + handle, + docs, + true, + lastPullAtRef.current, + ) + if (result.errors.length > 0) { + console.warn("Backup pull errors:", result.errors) + } + + let pulledAt = new Date().toISOString() + lastPullAtRef.current = Date.parse(pulledAt) + setLastPullAt(pulledAt) + } catch (e) { + console.error("Backup pull failed:", e) + } finally { + isPullingRef.current = false + } + } + + let watchAborted = false + let stopWatching: (() => void) | null = null + + async function setupWatch() { + let handle = await getBackupHandle() + if (!handle) return + + let stop = await observeDirectoryChanges(handle, () => { + if (!watchAborted) doPull() + }) + if (watchAborted) { + stop?.() + return + } + stopWatching = stop + doPull() + } + + setupWatch() + + return () => { + watchAborted = true + stopWatching?.() + } + }, [enabled, bidirectional, me, setLastPullAt]) + + return null +} + +function SpacesBackupSubscriber() { + let me = useAccount(UserAccount, { resolve: spacesBackupQuery }) + let [storageVersion, setStorageVersion] = useState(0) + + useEffect(() => { + function handleStorageChange(e: StorageEvent) { + if (e.key?.startsWith(SPACE_BACKUP_KEY_PREFIX)) { + setStorageVersion(v => v + 1) + } + } + + window.addEventListener("storage", handleStorageChange) + return () => window.removeEventListener("storage", handleStorageChange) + }, []) + + let spacesWithBackup = getSpacesWithBackup(me, storageVersion) + + return ( + <> + {spacesWithBackup.map(spaceId => ( + + ))} + + ) +} + +interface SpaceBackupSubscriberProps { + spaceId: string +} + +function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { + let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) + let debounceRef = useRef | null>(null) + let lastContentHashRef = useRef("") + let isPushingRef = useRef(false) + let isPullingRef = useRef(false) + + let space = useCoState(Space, spaceId, { + resolve: spaceBackupDocumentResolve, + }) + + useEffect(() => { + if (!directoryName) return + if (!space?.$isLoaded || !space.documents?.$isLoaded) return + + let docs = space.documents + let activeDocs = [...docs].filter(d => d?.$isLoaded && !d.deletedAt) + + let contentHash = activeDocs + .map(d => `${d.$jazz.id}:${d.updatedAt?.getTime()}`) + .sort() + .join("|") + + if (contentHash === lastContentHashRef.current) return + lastContentHashRef.current = contentHash + + if (debounceRef.current) clearTimeout(debounceRef.current) + debounceRef.current = setTimeout(async () => { + try { + let handle = await getSpaceBackupHandle(spaceId) + if (!handle) { + setDirectoryName(null) + return + } + + let loadedDocs = activeDocs.filter( + (d): d is LoadedDocument => d?.$isLoaded === true, + ) + let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) + let scopeId = docs.$jazz.id ? `docs:${docs.$jazz.id}` : "docs:unknown" + isPushingRef.current = true + await syncBackup(handle, backupDocs, scopeId) + } catch (e) { + console.error(`Space backup failed for ${spaceId}:`, e) + } finally { + isPushingRef.current = false + } + }, SPACE_BACKUP_DEBOUNCE_MS) + + return () => { + if (debounceRef.current) clearTimeout(debounceRef.current) + } + }, [directoryName, space, spaceId, setDirectoryName]) + + useEffect(() => { + if (!directoryName) return + if (!space?.$isLoaded || !space.documents?.$isLoaded) return + if (!supportsFileSystemWatch()) return + + let docs = space.documents + + let spaceGroup = + space.$jazz.owner instanceof Group ? space.$jazz.owner : null + let canWrite = + spaceGroup?.myRole() === "admin" || spaceGroup?.myRole() === "writer" + + async function doPull() { + try { + if (isPushingRef.current) return + if (isPullingRef.current) return + isPullingRef.current = true + let handle = await getSpaceBackupHandle(spaceId) + if (!handle) return + + if (!docs.$isLoaded || !isDocumentList(docs)) return + let result = await syncFromBackup( + handle, + docs, + canWrite, + spaceLastPullAtById.get(spaceId) ?? null, + ) + if (result.errors.length > 0) { + console.warn(`Space ${spaceId} pull errors:`, result.errors) + } + spaceLastPullAtById.set(spaceId, Date.now()) + } catch (e) { + console.error(`Space backup pull failed for ${spaceId}:`, e) + } finally { + isPullingRef.current = false + } + } + + let watchAborted = false + let stopWatching: (() => void) | null = null + + async function setupWatch() { + let handle = await getSpaceBackupHandle(spaceId) + if (!handle) return + + let stop = await observeDirectoryChanges(handle, () => { + if (!watchAborted) doPull() + }) + if (watchAborted) { + stop?.() + return + } + stopWatching = stop + doPull() + } + + setupWatch() + + return () => { + watchAborted = true + stopWatching?.() + } + }, [directoryName, space, spaceId]) + + return null +} + +function getSpacesWithBackup( + me: ReturnType< + typeof useAccount + >, + _storageVersion: number, +): string[] { + if (!me.$isLoaded || !me.root?.spaces?.$isLoaded) return [] + + let spaceIds: string[] = [] + for (let space of Array.from(me.root.spaces)) { + if (!space?.$isLoaded) continue + let backupPath = getSpaceBackupPath(space.$jazz.id) + if (backupPath) { + spaceIds.push(space.$jazz.id) + } + } + return spaceIds +} + +function isDocumentList(value: unknown): boolean { + if (typeof value !== "object" || value === null) return false + return "$jazz" in value && "find" in value +} diff --git a/src/lib/backup-sync.test.ts b/src/lib/backup-sync.test.ts new file mode 100644 index 0000000..3b16bc6 --- /dev/null +++ b/src/lib/backup-sync.test.ts @@ -0,0 +1,562 @@ +import { describe, it, expect } from "vitest" +import { + computeDocLocations, + transformContentForBackup, + computeExpectedStructure, + scanBackupFolder, + transformContentForImport, + readManifest, + writeManifest, + type BackupManifest, + type BackupDoc, +} from "./backup-sync" +import { + MockDirectoryHandle, + createMockBlob, + createMockFile, +} from "./backup-test-helpers" + +// ============================================================================= +// Compute Doc Locations +// ============================================================================= + +describe("computeDocLocations", () => { + function createDoc(input: { + id: string + title: string + path: string | null + assets?: { id: string; name: string; blob: Blob }[] + }): BackupDoc { + return { + id: input.id, + title: input.title, + content: "# Content", + path: input.path, + updatedAtMs: 0, + assets: input.assets ?? [], + } + } + + it("uses title.md for root docs without assets", () => { + let docs = [createDoc({ id: "d1", title: "Hello World", path: null })] + let locations = computeDocLocations(docs) + let loc = locations.get("d1") + + expect(loc?.dirPath).toBe("") + expect(loc?.filename).toBe("Hello World.md") + expect(loc?.hasOwnFolder).toBe(false) + }) + + it("creates doc folder for docs with assets", () => { + let docs = [ + createDoc({ + id: "d1", + title: "Project Note", + path: null, + assets: [{ id: "a1", name: "Image", blob: createMockBlob("img") }], + }), + ] + let locations = computeDocLocations(docs) + let loc = locations.get("d1") + + expect(loc?.dirPath).toBe("Project Note") + expect(loc?.filename).toBe("Project Note.md") + expect(loc?.hasOwnFolder).toBe(true) + }) + + it("disambiguates title collisions in same folder", () => { + let docs = [ + createDoc({ id: "doc-11111111", title: "Same", path: "work" }), + createDoc({ id: "doc-22222222", title: "Same", path: "work" }), + ] + let locations = computeDocLocations(docs) + let first = locations.get("doc-11111111") + let second = locations.get("doc-22222222") + + expect(first?.filename).toBe("Same.md") + expect(second?.filename).toContain("Same") + expect(second?.filename).toContain("22222222") + }) + + it("does not disambiguate same title in different folders", () => { + let docs = [ + createDoc({ id: "d1", title: "Same", path: "work" }), + createDoc({ id: "d2", title: "Same", path: "personal" }), + ] + let locations = computeDocLocations(docs) + + expect(locations.get("d1")?.filename).toBe("Same.md") + expect(locations.get("d2")?.filename).toBe("Same.md") + }) + + it("disambiguates duplicate asset filenames", () => { + let docs = [ + createDoc({ + id: "d1", + title: "Assets", + path: null, + assets: [ + { id: "a1", name: "shot", blob: createMockBlob("one") }, + { id: "a2", name: "shot", blob: createMockBlob("two") }, + ], + }), + ] + let locations = computeDocLocations(docs) + let loc = locations.get("d1") + + expect(loc?.assetFiles.get("a1")).toBeDefined() + expect(loc?.assetFiles.get("a2")).toBeDefined() + expect(loc?.assetFiles.get("a1")).not.toBe(loc?.assetFiles.get("a2")) + }) + + it("assigns stable duplicate asset filenames regardless of asset order", () => { + let docForward = createDoc({ + id: "d1", + title: "Assets", + path: null, + assets: [ + { id: "a1", name: "shot", blob: createMockBlob("one") }, + { id: "a2", name: "shot", blob: createMockBlob("two") }, + ], + }) + let docReverse = createDoc({ + id: "d1", + title: "Assets", + path: null, + assets: [ + { id: "a2", name: "shot", blob: createMockBlob("two") }, + { id: "a1", name: "shot", blob: createMockBlob("one") }, + ], + }) + + let forwardLoc = computeDocLocations([docForward]).get("d1") + let reverseLoc = computeDocLocations([docReverse]).get("d1") + + expect(forwardLoc?.assetFiles.get("a1")).toBe( + reverseLoc?.assetFiles.get("a1"), + ) + expect(forwardLoc?.assetFiles.get("a2")).toBe( + reverseLoc?.assetFiles.get("a2"), + ) + }) + + it("assigns stable collision filenames regardless of input order", () => { + let first: BackupDoc = { + id: "doc-aaaa1111", + title: "Same", + content: "x", + path: "work", + updatedAtMs: 0, + assets: [], + } + let second: BackupDoc = { + id: "doc-bbbb2222", + title: "Same", + content: "x", + path: "work", + updatedAtMs: 0, + assets: [], + } + + let forward = computeDocLocations([first, second]) + let reverse = computeDocLocations([second, first]) + + expect(forward.get(first.id)?.filename).toBe( + reverse.get(first.id)?.filename, + ) + expect(forward.get(second.id)?.filename).toBe( + reverse.get(second.id)?.filename, + ) + }) +}) + +// ============================================================================= +// Transform Content for Backup +// ============================================================================= + +describe("transformContentForBackup", () => { + it("transforms asset: references to assets paths", () => { + let assetFiles = new Map([ + ["asset1", "photo.png"], + ["asset2", "clip.jpg"], + ]) + let content = "![A](asset:asset1)\n![B](asset:asset2)" + + let result = transformContentForBackup(content, assetFiles) + + expect(result).toContain("![A](assets/photo.png)") + expect(result).toContain("![B](assets/clip.jpg)") + }) + + it("keeps unmatched asset references unchanged", () => { + let assetFiles = new Map([["asset1", "photo.png"]]) + let content = "![A](asset:missing)" + + let result = transformContentForBackup(content, assetFiles) + + expect(result).toBe("![A](asset:missing)") + }) +}) + +// ============================================================================= +// Compute Expected Structure +// ============================================================================= + +describe("computeExpectedStructure", () => { + it("includes parent directories and markdown files", () => { + let docs: BackupDoc[] = [ + { + id: "d1", + title: "Note", + content: "x", + path: "work/notes", + updatedAtMs: 0, + assets: [], + }, + ] + let locations = computeDocLocations(docs) + let expected = computeExpectedStructure(docs, locations) + + expect(expected.expectedPaths.has("work")).toBe(true) + expect(expected.expectedPaths.has("work/notes")).toBe(true) + expect(expected.expectedFiles.get("work/notes")?.has("Note.md")).toBe(true) + }) + + it("includes assets folder for docs with assets", () => { + let docs: BackupDoc[] = [ + { + id: "d1", + title: "Note", + content: "x", + path: "work", + updatedAtMs: 0, + assets: [{ id: "a1", name: "image", blob: createMockBlob("img") }], + }, + ] + let locations = computeDocLocations(docs) + let expected = computeExpectedStructure(docs, locations) + + expect(expected.expectedPaths.has("work/Note")).toBe(true) + expect(expected.expectedPaths.has("work/Note/assets")).toBe(true) + }) +}) + +// ============================================================================= +// Transform Content for Import +// ============================================================================= + +describe("transformContentForImport", () => { + it("transforms asset paths to asset: references", () => { + let assetFiles = new Map([ + ["asset1", "image.png"], + ["asset2", "photo.jpg"], + ]) + + let content = `# Test + +![Screenshot](assets/image.png) +![Photo](assets/photo.jpg) +` + + let result = transformContentForImport(content, assetFiles) + + expect(result).toContain("![Screenshot](asset:asset1)") + expect(result).toContain("![Photo](asset:asset2)") + }) + + it("preserves non-asset references", () => { + let assetFiles = new Map([["asset1", "image.png"]]) + + let content = `# Test + +![External](https://example.com/img.png) +![Local](./local/image.png) +![Asset](assets/image.png) +` + + let result = transformContentForImport(content, assetFiles) + + expect(result).toContain("https://example.com/img.png") + expect(result).toContain("./local/image.png") + expect(result).toContain("asset:asset1") + }) + + it("keeps unknown asset paths unchanged", () => { + let assetFiles = new Map([["asset1", "image.png"]]) + + let content = `![Unknown](assets/unknown.jpg)` + + let result = transformContentForImport(content, assetFiles) + + expect(result).toBe(`![Unknown](assets/unknown.jpg)`) + }) +}) + +// ============================================================================= +// Manifest Read/Write +// ============================================================================= + +describe("readManifest", () => { + it("returns null when manifest file does not exist", async () => { + let root = new MockDirectoryHandle("root") + + let result = await readManifest(root) + + expect(result).toBeNull() + }) + + it("reads and parses valid manifest", async () => { + let root = new MockDirectoryHandle("root") + let manifest: BackupManifest = { + version: 1, + entries: [ + { + docId: "doc1", + relativePath: "Test.md", + contentHash: "abc123", + lastSyncedAt: new Date().toISOString(), + assets: [], + }, + ], + lastSyncAt: new Date().toISOString(), + } + + root.addFile(".alkalye-manifest.json", JSON.stringify(manifest)) + + let result = await readManifest(root) + + expect(result).not.toBeNull() + expect(result!.version).toBe(1) + expect(result!.entries).toHaveLength(1) + expect(result!.entries[0].docId).toBe("doc1") + }) + + it("returns null for invalid version", async () => { + let root = new MockDirectoryHandle("root") + let invalidManifest = { version: 2, entries: [] } + + root.addFile(".alkalye-manifest.json", JSON.stringify(invalidManifest)) + + let result = await readManifest(root) + + expect(result).toBeNull() + }) + + it("returns null for invalid entries shape", async () => { + let root = new MockDirectoryHandle("root") + let invalidManifest = { + version: 1, + entries: [{ docId: "d1" }], + lastSyncAt: new Date().toISOString(), + } + + root.addFile(".alkalye-manifest.json", JSON.stringify(invalidManifest)) + + let result = await readManifest(root) + + expect(result).toBeNull() + }) + + it("returns null for malformed JSON", async () => { + let root = new MockDirectoryHandle("root") + root.addFile(".alkalye-manifest.json", "{invalid") + + let result = await readManifest(root) + + expect(result).toBeNull() + }) +}) + +describe("writeManifest", () => { + it("writes manifest to directory", async () => { + let root = new MockDirectoryHandle("root") + let manifest: BackupManifest = { + version: 1, + entries: [ + { + docId: "doc1", + relativePath: "Test.md", + contentHash: "abc123", + lastSyncedAt: "2024-01-01T00:00:00Z", + assets: [], + }, + ], + lastSyncAt: "2024-01-01T00:00:00Z", + } + + await writeManifest(root, manifest) + + let written = await readManifest(root) + expect(written).not.toBeNull() + expect(written!.entries[0].docId).toBe("doc1") + }) +}) + +// ============================================================================= +// Scan Backup Folder +// ============================================================================= + +describe("scanBackupFolder", () => { + it("scans markdown files at root level", async () => { + let root = new MockDirectoryHandle("root") + root.addFile("Test.md", createMockFile("# Test Content")) + root.addFile("Another.md", createMockFile("# Another")) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(2) + let names = files.map(f => f.name).sort() + expect(names).toEqual(["Another", "Test"]) + }) + + it("recursively scans nested directories", async () => { + let root = new MockDirectoryHandle("root") + let workDir = new MockDirectoryHandle("work") + let notesDir = new MockDirectoryHandle("notes") + + workDir.addDirectory("notes", notesDir) + root.addDirectory("work", workDir) + + root.addFile("Root.md", createMockFile("# Root")) + workDir.addFile("Work.md", createMockFile("# Work")) + notesDir.addFile("Notes.md", createMockFile("# Notes")) + + let files = await scanBackupFolder(root) + + let paths = files.map(f => f.relativePath).sort() + expect(paths).toEqual(["Root.md", "work/Work.md", "work/notes/Notes.md"]) + }) + + it("returns files in stable lexicographic order", async () => { + let root = new MockDirectoryHandle("root") + root.addFile("z-last.md", createMockFile("# z")) + root.addFile("a-first.md", createMockFile("# a")) + + let files = await scanBackupFolder(root) + + expect(files.map(file => file.relativePath)).toEqual([ + "a-first.md", + "z-last.md", + ]) + }) + + it("scans markdown extension case-insensitively", async () => { + let root = new MockDirectoryHandle("root") + root.addFile("UPPER.MD", createMockFile("# Upper")) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(1) + expect(files[0].name).toBe("UPPER") + }) + + it("collects assets from assets folders", async () => { + let root = new MockDirectoryHandle("root") + let docDir = new MockDirectoryHandle("My Doc") + + root.addDirectory("My Doc", docDir) + let assetsDir = new MockDirectoryHandle("assets") + docDir.addDirectory("assets", assetsDir) + + docDir.addFile( + "My Doc.md", + createMockFile("# My Doc\n\n![Image](assets/photo.png)"), + ) + assetsDir.addFile("photo.png", "image data") + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(1) + expect(files[0].assets).toHaveLength(1) + expect(files[0].assets[0].name).toBe("photo.png") + }) + + it("loads only referenced assets for each markdown file", async () => { + let root = new MockDirectoryHandle("root") + root.addFile("Doc One.md", createMockFile("![One](assets/one.png)")) + root.addFile("Doc Two.md", createMockFile("No assets here")) + + let assetsDir = new MockDirectoryHandle("assets") + assetsDir.addFile("one.png", "1") + assetsDir.addFile("two.png", "2") + root.addDirectory("assets", assetsDir) + + let files = await scanBackupFolder(root) + let byName = new Map(files.map(file => [file.name, file])) + + expect(byName.get("Doc One")?.assets.map(asset => asset.name)).toEqual([ + "one.png", + ]) + expect(byName.get("Doc Two")?.assets).toHaveLength(0) + }) + + it("does not import markdown files inside assets directories", async () => { + let root = new MockDirectoryHandle("root") + let docDir = new MockDirectoryHandle("Doc") + let assetsDir = new MockDirectoryHandle("assets") + docDir.addDirectory("assets", assetsDir) + root.addDirectory("Doc", docDir) + docDir.addFile("Doc.md", createMockFile("# Doc")) + assetsDir.addFile("notes.md", createMockFile("# Should stay asset")) + + let files = await scanBackupFolder(root) + + expect(files.map(file => file.relativePath)).toEqual(["Doc/Doc.md"]) + }) + + it("scans docs inside top-level assets folder when it is a normal directory", async () => { + let root = new MockDirectoryHandle("root") + let assetsDir = new MockDirectoryHandle("assets") + root.addDirectory("assets", assetsDir) + assetsDir.addFile("Roadmap.md", createMockFile("# Roadmap")) + + let files = await scanBackupFolder(root) + + expect(files.map(file => file.relativePath)).toEqual(["assets/Roadmap.md"]) + }) + + it("skips dot directories", async () => { + let root = new MockDirectoryHandle("root") + let hiddenDir = new MockDirectoryHandle(".hidden") + + root.addDirectory(".hidden", hiddenDir) + hiddenDir.addFile("Hidden.md", createMockFile("# Hidden")) + root.addFile("Visible.md", createMockFile("# Visible")) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(1) + expect(files[0].name).toBe("Visible") + }) + + it("skips manifest file", async () => { + let root = new MockDirectoryHandle("root") + root.addFile(".alkalye-manifest.json", '{"version":1}') + root.addFile("Test.md", createMockFile("# Test")) + + let files = await scanBackupFolder(root) + + expect(files).toHaveLength(1) + expect(files[0].name).toBe("Test") + }) + + it("skips hidden markdown files", async () => { + let root = new MockDirectoryHandle("root") + root.addFile(".hidden.md", createMockFile("# Hidden")) + root.addFile("Visible.md", createMockFile("# Visible")) + + let files = await scanBackupFolder(root) + + expect(files.map(file => file.relativePath)).toEqual(["Visible.md"]) + }) + + it("captures lastModified timestamp", async () => { + let root = new MockDirectoryHandle("root") + let timestamp = 1234567890000 + root.addFile("Test.md", createMockFile("# Test"), timestamp) + + let files = await scanBackupFolder(root) + + expect(files[0].lastModified).toBe(timestamp) + }) +}) diff --git a/src/lib/backup-sync.ts b/src/lib/backup-sync.ts index ca0ad99..c126b99 100644 --- a/src/lib/backup-sync.ts +++ b/src/lib/backup-sync.ts @@ -1,12 +1,20 @@ import { getExtensionFromBlob, sanitizeFilename } from "@/lib/export" +import { z } from "zod" export { computeDocLocations, transformContentForBackup, computeExpectedStructure, + transformContentForImport, + scanBackupFolder, + readManifest, + writeManifest, type BackupDoc, type DocLocation, type ExpectedStructure, + type BackupManifest, + type ManifestEntry, + type ScannedFile, } interface BackupDoc { @@ -14,6 +22,7 @@ interface BackupDoc { title: string content: string path: string | null + updatedAtMs: number assets: { id: string; name: string; blob: Blob }[] } @@ -29,16 +38,76 @@ interface ExpectedStructure { expectedFiles: Map> } +interface ManifestEntry { + docId: string + relativePath: string + scopeId?: string + locationKey?: string + contentHash: string + lastSyncedAt: string + assets: { id?: string; name: string; hash: string }[] +} + +interface BackupManifest { + version: 1 + entries: ManifestEntry[] + lastSyncAt: string +} + +interface ScannedFile { + relativePath: string + name: string + content: string + assets: { name: string; blob: Blob }[] + lastModified: number +} + +let manifestAssetSchema = z.object({ + id: z.string().optional(), + name: z.string(), + hash: z.string(), +}) + +let manifestEntrySchema = z.object({ + docId: z.string(), + relativePath: z.string(), + scopeId: z.string().optional(), + locationKey: z.string().optional(), + contentHash: z.string(), + lastSyncedAt: z.string(), + assets: z.array(manifestAssetSchema), +}) + +let backupManifestSchema = z.object({ + version: z.literal(1), + entries: z.array(manifestEntrySchema), + lastSyncAt: z.string(), +}) + function computeDocLocations(docs: BackupDoc[]): Map { let docLocations = new Map() - let usedNames = new Map>() // parentPath -> used names (lowercase) + let usedNames = new Map>() + let sortedDocs = [...docs].sort((left, right) => { + let leftParentPath = left.path ?? "" + let rightParentPath = right.path ?? "" + if (leftParentPath !== rightParentPath) { + return leftParentPath.localeCompare(rightParentPath) + } - for (let doc of docs) { + let leftName = sanitizeFilename(left.title) + let rightName = sanitizeFilename(right.title) + if (leftName !== rightName) { + return leftName.localeCompare(rightName) + } + + return left.id.localeCompare(right.id) + }) + + for (let doc of sortedDocs) { let baseName = sanitizeFilename(doc.title) let hasAssets = doc.assets.length > 0 let parentPath = doc.path ?? "" - // Track used names at parent level for conflict detection if (!usedNames.has(parentPath)) usedNames.set(parentPath, new Set()) let used = usedNames.get(parentPath)! @@ -60,10 +129,17 @@ function computeDocLocations(docs: BackupDoc[]): Map { hasOwnFolder = false } - // Build asset filename map for this doc let assetFiles = new Map() let usedAssetNames = new Set() - for (let asset of doc.assets) { + let sortedAssets = [...doc.assets].sort((left, right) => { + let leftName = sanitizeFilename(left.name) || "image" + let rightName = sanitizeFilename(right.name) || "image" + if (leftName !== rightName) { + return leftName.localeCompare(rightName) + } + return left.id.localeCompare(right.id) + }) + for (let asset of sortedAssets) { let ext = getExtensionFromBlob(asset.blob) let assetBaseName = sanitizeFilename(asset.name) || "image" let fileName = assetBaseName + ext @@ -102,6 +178,23 @@ function transformContentForBackup( ) } +function transformContentForImport( + content: string, + assetFiles: Map, +): string { + return content.replace( + /!\[([^\]]*)\]\(assets\/([^)]+)\)/g, + (match, alt, assetFilename) => { + for (let [id, filename] of assetFiles) { + if (filename === assetFilename) { + return `![${alt}](asset:${id})` + } + } + return match + }, + ) +} + function computeExpectedStructure( docs: BackupDoc[], docLocations: Map, @@ -112,7 +205,6 @@ function computeExpectedStructure( for (let doc of docs) { let loc = docLocations.get(doc.id)! - // Add the directory path and all parent paths if (loc.dirPath) { let parts = loc.dirPath.split("/") for (let i = 1; i <= parts.length; i++) { @@ -120,13 +212,11 @@ function computeExpectedStructure( } } - // Add expected file if (!expectedFiles.has(loc.dirPath)) { expectedFiles.set(loc.dirPath, new Set()) } expectedFiles.get(loc.dirPath)!.add(loc.filename) - // If doc has assets, expect assets subfolder if (loc.hasOwnFolder && doc.assets.length > 0) { let assetsPath = loc.dirPath ? `${loc.dirPath}/assets` : "assets" expectedPaths.add(assetsPath) @@ -135,3 +225,127 @@ function computeExpectedStructure( return { expectedPaths, expectedFiles } } + +async function scanBackupFolder( + handle: FileSystemDirectoryHandle, +): Promise { + let files: ScannedFile[] = [] + + function getReferencedAssetNames(content: string): Set { + let references = new Set() + for (let match of content.matchAll(/!\[[^\]]*\]\(assets\/([^)]+)\)/g)) { + let filename = match[1] + if (!filename) continue + references.add(filename) + } + return references + } + + async function scanDir( + dir: FileSystemDirectoryHandle, + relativePath: string, + ): Promise { + async function isGeneratedAssetsDirectory( + parentDir: FileSystemDirectoryHandle, + parentRelativePath: string, + ): Promise { + let parentName = parentRelativePath.split("/").filter(Boolean).at(-1) + if (!parentName) return false + + try { + await parentDir.getFileHandle(`${parentName}.md`) + return true + } catch { + return false + } + } + + for await (let [name, handle] of dir.entries()) { + let entryPath = relativePath ? `${relativePath}/${name}` : name + let loweredName = name.toLowerCase() + + if (handle.kind === "directory") { + if (name.startsWith(".")) continue + if (loweredName === "assets") { + let isDocAssetsDirectory = await isGeneratedAssetsDirectory( + dir, + relativePath, + ) + if (isDocAssetsDirectory) continue + } + let subDir = await dir.getDirectoryHandle(name) + await scanDir(subDir, entryPath) + } else if (handle.kind === "file" && loweredName.endsWith(".md")) { + if (name.startsWith(".")) continue + if (name === ".alkalye-manifest.json") continue + + let fileHandle = await dir.getFileHandle(name) + let file = await fileHandle.getFile() + let content = await file.text() + let lastModified = file.lastModified + + let assets: { name: string; blob: Blob }[] = [] + let referencedAssets = getReferencedAssetNames(content) + if (referencedAssets.size > 0) { + let assetsDir = await dir + .getDirectoryHandle("assets") + .catch(() => null) + if (assetsDir) { + for (let assetName of referencedAssets) { + if (assetName.startsWith(".")) continue + try { + let assetFileHandle = await assetsDir.getFileHandle(assetName) + let assetFile = await assetFileHandle.getFile() + assets.push({ name: assetName, blob: assetFile }) + } catch { + continue + } + } + } + } + + files.push({ + relativePath: entryPath, + name: name.replace(/\.md$/i, ""), + content, + assets, + lastModified, + }) + } + } + } + + await scanDir(handle, "") + files.sort((left, right) => + left.relativePath.localeCompare(right.relativePath), + ) + return files +} + +async function readManifest( + handle: FileSystemDirectoryHandle, +): Promise { + try { + let fileHandle = await handle.getFileHandle(".alkalye-manifest.json") + let file = await fileHandle.getFile() + let text = await file.text() + let parsed = JSON.parse(text) + let validated = backupManifestSchema.safeParse(parsed) + if (!validated.success) return null + return validated.data + } catch { + return null + } +} + +async function writeManifest( + handle: FileSystemDirectoryHandle, + manifest: BackupManifest, +): Promise { + let fileHandle = await handle.getFileHandle(".alkalye-manifest.json", { + create: true, + }) + let writable = await fileHandle.createWritable() + await writable.write(JSON.stringify(manifest, null, 2)) + await writable.close() +} diff --git a/src/lib/backup-test-helpers.ts b/src/lib/backup-test-helpers.ts new file mode 100644 index 0000000..ff15da4 --- /dev/null +++ b/src/lib/backup-test-helpers.ts @@ -0,0 +1,485 @@ +export { + MockDirectoryHandle, + createMockBlob, + createMockFile, + readFileAtPath, + removeFileAtPath, + writeFileAtPath, + basename, +} + +interface StoredFile { + content: string | null + source: File | null + lastModified: number + type: string +} + +function createMockFile(content: string): string { + return content +} + +class MockBlob implements Blob { + size: number + type: string + private data: Uint8Array + + constructor(content: string | Uint8Array, type: string) { + let raw = typeof content === "string" ? encodeText(content) : content + this.data = toStrictBytes(raw) + this.size = this.data.byteLength + this.type = type + } + + bytes(): Promise> { + return Promise.resolve(toStrictBytes(this.data)) + } + + arrayBuffer(): Promise { + return Promise.resolve(bytesToArrayBuffer(this.data)) + } + + text(): Promise { + return Promise.resolve(decodeBytes(this.data)) + } + + stream(): ReadableStream> { + let bytes = toStrictBytes(this.data) + return new ReadableStream>({ + start(controller) { + controller.enqueue(bytes) + controller.close() + }, + }) + } + + slice(start?: number, end?: number, contentType?: string): Blob { + let next = this.data.slice(start ?? 0, end ?? this.data.byteLength) + return new MockBlob(next, contentType ?? this.type) + } + + get [Symbol.toStringTag](): string { + return "Blob" + } +} + +function createMockBlob(content: string, type = "image/png"): Blob { + return new MockBlob(content, type) +} + +async function readFileAtPath( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + + let fileHandle = await dir.getFileHandle(parts[parts.length - 1]) + let file = await fileHandle.getFile() + return file.text() +} + +async function removeFileAtPath( + root: MockDirectoryHandle, + relativePath: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i]) + } + + await dir.removeEntry(parts[parts.length - 1]) +} + +async function writeFileAtPath( + root: MockDirectoryHandle, + relativePath: string, + content: string, +): Promise { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) throw new Error("Empty path") + + let dir: FileSystemDirectoryHandle = root + for (let i = 0; i < parts.length - 1; i++) { + dir = await dir.getDirectoryHandle(parts[i], { create: true }) + } + + let fileHandle = await dir.getFileHandle(parts[parts.length - 1], { + create: true, + }) + let writable = await fileHandle.createWritable() + await writable.write(content) + await writable.close() +} + +function basename(relativePath: string): string { + let parts = relativePath.split("/").filter(Boolean) + if (parts.length === 0) return relativePath + return parts[parts.length - 1] +} + +function encodeText(value: string): Uint8Array { + return new TextEncoder().encode(value) +} + +function bytesToArrayBuffer(bytes: Uint8Array): ArrayBuffer { + let buffer = new ArrayBuffer(bytes.byteLength) + new Uint8Array(buffer).set(bytes) + return buffer +} + +function toStrictBytes(bytes: Uint8Array): Uint8Array { + let buffer = bytesToArrayBuffer(bytes) + let strict: Uint8Array = new Uint8Array(buffer) + return strict +} + +function decodeBytes(bytes: Uint8Array): string { + return new TextDecoder().decode(bytes) +} + +async function readBlobBytes(blob: Blob): Promise> { + if (typeof blob.arrayBuffer === "function") { + let buffer = await blob.arrayBuffer() + return new Uint8Array(buffer) + } + + if (typeof blob.text === "function") { + let text = await blob.text() + return encodeText(text) + } + + throw new Error("Blob cannot be read in this environment") +} + +class MockWritableFileStream implements FileSystemWritableFileStream { + private stream = new WritableStream() + private saveContent: (content: string) => void + + constructor(saveContent: (content: string) => void) { + this.saveContent = saveContent + } + + get locked(): boolean { + return this.stream.locked + } + + abort(reason?: unknown): Promise { + return this.stream.abort(reason) + } + + close(): Promise { + return Promise.resolve() + } + + getWriter(): WritableStreamDefaultWriter { + return this.stream.getWriter() + } + + seek(_position: number): Promise { + return Promise.resolve() + } + + truncate(_size: number): Promise { + return Promise.resolve() + } + + write(data: string | Blob | ArrayBuffer): Promise + write(data: ArrayBufferView): Promise + write(data: { + type: "write" + data: string | Blob | ArrayBuffer | ArrayBufferView | null + }): Promise + write(data: { type: "seek"; position: number }): Promise + write(data: { type: "truncate"; size: number }): Promise + async write(data: unknown): Promise { + if (typeof data === "string") { + this.saveContent(data) + return + } + + if (data instanceof Blob) { + this.saveContent(await data.text()) + return + } + + if (data instanceof ArrayBuffer) { + let bytes = new Uint8Array(data) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + + if (ArrayBuffer.isView(data)) { + let bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + + if (typeof data === "object" && data !== null && "type" in data) { + if ( + data.type === "write" && + "data" in data && + data.data !== undefined && + data.data !== null + ) { + let nestedData = data.data + if (typeof nestedData === "string") { + this.saveContent(nestedData) + return + } + if (nestedData instanceof Blob) { + this.saveContent(await nestedData.text()) + return + } + if (nestedData instanceof ArrayBuffer) { + let bytes = new Uint8Array(nestedData) + this.saveContent(new TextDecoder().decode(bytes)) + return + } + if (ArrayBuffer.isView(nestedData)) { + let bytes = new Uint8Array( + nestedData.buffer, + nestedData.byteOffset, + nestedData.byteLength, + ) + this.saveContent(new TextDecoder().decode(bytes)) + } + } + } + } + + get [Symbol.toStringTag](): string { + return "FileSystemWritableFileStream" + } +} + +class MockFileHandle implements FileSystemFileHandle { + kind = "file" as const + name: string + private getStoredFile: () => StoredFile | undefined + private setStoredFile: (file: StoredFile) => void + + constructor( + name: string, + getStoredFile: () => StoredFile | undefined, + setStoredFile: (file: StoredFile) => void, + ) { + this.name = name + this.getStoredFile = getStoredFile + this.setStoredFile = setStoredFile + } + + async getFile(): Promise { + let file = this.getStoredFile() + if (!file) { + return new File([""], this.name) + } + + let bytes = + file.content !== null ? encodeText(file.content) : new Uint8Array(0) + if (file.content === null && file.source) { + bytes = await readBlobBytes(file.source) + } + + let compatibleFile = new File([bytesToArrayBuffer(bytes)], this.name, { + lastModified: file.lastModified, + type: file.type, + }) + + if (typeof compatibleFile.text !== "function") { + Object.defineProperty(compatibleFile, "text", { + value: async () => decodeBytes(bytes), + }) + } + if (typeof compatibleFile.arrayBuffer !== "function") { + Object.defineProperty(compatibleFile, "arrayBuffer", { + value: async () => bytesToArrayBuffer(bytes), + }) + } + + return compatibleFile + } + + async createWritable(): Promise { + return new MockWritableFileStream(content => { + let current = this.getStoredFile() + this.setStoredFile({ + content, + source: null, + lastModified: Date.now(), + type: current?.type ?? inferMimeType(this.name), + }) + }) + } + + isSameEntry(other: FileSystemHandle): Promise { + return Promise.resolve(other.kind === "file" && other.name === this.name) + } + + get [Symbol.toStringTag](): string { + return "FileSystemFileHandle" + } +} + +function isFileHandle( + handle: FileSystemHandle | undefined, +): handle is FileSystemFileHandle { + return handle?.kind === "file" +} + +function isDirectoryHandle( + handle: FileSystemHandle | undefined, +): handle is FileSystemDirectoryHandle { + return handle?.kind === "directory" +} + +class MockDirectoryHandle implements FileSystemDirectoryHandle { + kind = "directory" as const + name: string + private children = new Map() + private files = new Map() + + constructor(name: string) { + this.name = name + } + + addFile( + name: string, + fileOrContent: File | string, + lastModified = Date.now(), + ) { + let file = + typeof fileOrContent === "string" + ? { + content: fileOrContent, + source: null, + lastModified, + type: inferMimeType(name), + } + : { + content: null, + source: fileOrContent, + lastModified: fileOrContent.lastModified || lastModified, + type: fileOrContent.type || inferMimeType(name), + } + + this.files.set(name, file) + this.children.set( + name, + new MockFileHandle( + name, + () => this.files.get(name), + updated => this.files.set(name, updated), + ), + ) + } + + addDirectory(name: string, directory: MockDirectoryHandle) { + this.children.set(name, directory) + } + + entries(): AsyncIterableIterator<[string, FileSystemHandle]> { + let iter = this.children.entries() + return { + async next() { + let result = iter.next() + if (result.done) return { done: true, value: undefined } + return { done: false, value: result.value } + }, + [Symbol.asyncIterator]() { + return this + }, + } + } + + async getFileHandle( + name: string, + options?: { create?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!isFileHandle(handle)) { + if (options?.create) { + this.addFile(name, "") + let created = this.children.get(name) + if (!isFileHandle(created)) throw new Error("Failed creating file") + return created + } + throw new Error(`File not found: ${name}`) + } + return handle + } + + async getDirectoryHandle( + name: string, + options?: { create?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!isDirectoryHandle(handle)) { + if (options?.create) { + let created = new MockDirectoryHandle(name) + this.children.set(name, created) + return created + } + throw new Error(`Directory not found: ${name}`) + } + return handle + } + + async removeEntry( + name: string, + options?: { recursive?: boolean }, + ): Promise { + let handle = this.children.get(name) + if (!handle) return + + if (handle.kind === "directory" && !options?.recursive) { + throw new Error("Directory removal requires recursive flag") + } + + this.children.delete(name) + this.files.delete(name) + } + + resolve(): Promise { + return Promise.resolve([this.name]) + } + + queryPermission(): Promise<"granted"> { + return Promise.resolve("granted") + } + + requestPermission(): Promise<"granted"> { + return Promise.resolve("granted") + } + + isSameEntry(other: FileSystemHandle): Promise { + return Promise.resolve( + other.kind === "directory" && other.name === this.name, + ) + } + + get [Symbol.toStringTag](): string { + return "FileSystemDirectoryHandle" + } +} + +function inferMimeType(filename: string): string { + let lowered = filename.toLowerCase() + if (lowered.endsWith(".mp4")) return "video/mp4" + if (lowered.endsWith(".webm")) return "video/webm" + if (lowered.endsWith(".png")) return "image/png" + if (lowered.endsWith(".jpg") || lowered.endsWith(".jpeg")) return "image/jpeg" + if (lowered.endsWith(".gif")) return "image/gif" + if (lowered.endsWith(".webp")) return "image/webp" + if (lowered.endsWith(".svg")) return "image/svg+xml" + if (lowered.endsWith(".md")) return "text/markdown" + return "application/octet-stream" +} diff --git a/src/lib/backup.test.ts b/src/lib/backup.test.ts index 5a00c59..de8afd0 100644 --- a/src/lib/backup.test.ts +++ b/src/lib/backup.test.ts @@ -1,423 +1,52 @@ import { describe, it, expect } from "vitest" -import { - computeDocLocations, - transformContentForBackup, - computeExpectedStructure, - type BackupDoc, -} from "./backup-sync" +import { hashContent, syncFromBackup } from "./backup-engine" -// ============================================================================= -// Test Helpers -// ============================================================================= +describe("hashContent", () => { + it("returns consistent hash for same content", async () => { + let content = "Hello, World!" + let hash1 = await hashContent(content) + let hash2 = await hashContent(content) -function createBackupDoc( - overrides: Partial & { id: string }, -): BackupDoc { - return { - title: "Test Doc", - content: "# Test\n\nContent", - path: null, - assets: [], - ...overrides, - } -} - -function createBlob(type = "image/png"): Blob { - return new Blob([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], { type }) -} - -// ============================================================================= -// Document Location Computation -// ============================================================================= - -describe("computeDocLocations", () => { - describe("docs without assets", () => { - it("places doc at root with no path", () => { - let docs = [createBackupDoc({ id: "doc1", title: "My Doc" })] - let locations = computeDocLocations(docs) - - expect(locations.get("doc1")).toEqual({ - dirPath: "", - filename: "My Doc.md", - hasOwnFolder: false, - assetFiles: new Map(), - }) - }) - - it("places doc in path directory", () => { - let docs = [ - createBackupDoc({ id: "doc1", title: "My Doc", path: "work/notes" }), - ] - let locations = computeDocLocations(docs) - - expect(locations.get("doc1")).toEqual({ - dirPath: "work/notes", - filename: "My Doc.md", - hasOwnFolder: false, - assetFiles: new Map(), - }) - }) - }) - - describe("docs with assets", () => { - it("creates own folder at root", () => { - let docs = [ - createBackupDoc({ - id: "doc1", - title: "My Doc", - assets: [{ id: "asset1", name: "image", blob: createBlob() }], - }), - ] - let locations = computeDocLocations(docs) - - let loc = locations.get("doc1")! - expect(loc.dirPath).toBe("My Doc") - expect(loc.filename).toBe("My Doc.md") - expect(loc.hasOwnFolder).toBe(true) - expect(loc.assetFiles.get("asset1")).toBe("image.png") - }) - - it("creates own folder within path", () => { - let docs = [ - createBackupDoc({ - id: "doc1", - title: "My Doc", - path: "work/notes", - assets: [{ id: "asset1", name: "image", blob: createBlob() }], - }), - ] - let locations = computeDocLocations(docs) - - let loc = locations.get("doc1")! - expect(loc.dirPath).toBe("work/notes/My Doc") - expect(loc.filename).toBe("My Doc.md") - expect(loc.hasOwnFolder).toBe(true) - }) - }) - - describe("name conflict handling", () => { - it("adds short id suffix for duplicate names at same level", () => { - let docs = [ - createBackupDoc({ id: "doc1_abcd1234", title: "Notes" }), - createBackupDoc({ id: "doc2_efgh5678", title: "Notes" }), - ] - let locations = computeDocLocations(docs) - - expect(locations.get("doc1_abcd1234")?.filename).toBe("Notes.md") - expect(locations.get("doc2_efgh5678")?.filename).toBe( - "Notes (efgh5678).md", - ) - }) - - it("allows same name in different paths", () => { - let docs = [ - createBackupDoc({ id: "doc1", title: "Notes", path: "work" }), - createBackupDoc({ id: "doc2", title: "Notes", path: "personal" }), - ] - let locations = computeDocLocations(docs) - - expect(locations.get("doc1")?.filename).toBe("Notes.md") - expect(locations.get("doc2")?.filename).toBe("Notes.md") - }) - - it("handles case-insensitive conflicts", () => { - let docs = [ - createBackupDoc({ id: "doc1_aaaaaaaa", title: "Notes" }), - createBackupDoc({ id: "doc2_bbbbbbbb", title: "NOTES" }), - ] - let locations = computeDocLocations(docs) - - expect(locations.get("doc1_aaaaaaaa")?.filename).toBe("Notes.md") - expect(locations.get("doc2_bbbbbbbb")?.filename).toBe( - "NOTES (bbbbbbbb).md", - ) - }) - }) - - describe("asset filename handling", () => { - it("uses correct extension from blob type", () => { - let docs = [ - createBackupDoc({ - id: "doc1", - title: "Doc", - assets: [ - { id: "a1", name: "photo", blob: createBlob("image/jpeg") }, - { id: "a2", name: "icon", blob: createBlob("image/svg+xml") }, - { id: "a3", name: "pic", blob: createBlob("image/webp") }, - ], - }), - ] - let locations = computeDocLocations(docs) - - let loc = locations.get("doc1")! - expect(loc.assetFiles.get("a1")).toBe("photo.jpg") - expect(loc.assetFiles.get("a2")).toBe("icon.svg") - expect(loc.assetFiles.get("a3")).toBe("pic.webp") - }) - - it("handles duplicate asset names with counters", () => { - let docs = [ - createBackupDoc({ - id: "doc1", - title: "Doc", - assets: [ - { id: "a1", name: "image", blob: createBlob() }, - { id: "a2", name: "image", blob: createBlob() }, - { id: "a3", name: "image", blob: createBlob() }, - ], - }), - ] - let locations = computeDocLocations(docs) - - let loc = locations.get("doc1")! - expect(loc.assetFiles.get("a1")).toBe("image.png") - expect(loc.assetFiles.get("a2")).toBe("image-1.png") - expect(loc.assetFiles.get("a3")).toBe("image-2.png") - }) - - it("sanitizes asset filenames", () => { - let docs = [ - createBackupDoc({ - id: "doc1", - title: "Doc", - assets: [ - { id: "a1", name: "my:file/name", blob: createBlob() }, - { id: "a2", name: "", blob: createBlob() }, - ], - }), - ] - let locations = computeDocLocations(docs) - - let loc = locations.get("doc1")! - expect(loc.assetFiles.get("a1")).toBe("my_file_name.png") - // Empty name gets sanitized to "untitled" by sanitizeFilename - expect(loc.assetFiles.get("a2")).toBe("untitled.png") - }) - }) - - describe("title sanitization", () => { - it("sanitizes filesystem-unsafe characters", () => { - let docs = [ - createBackupDoc({ - id: "doc1", - title: 'What: "A Title?" ', - }), - ] - let locations = computeDocLocations(docs) - - expect(locations.get("doc1")?.filename).toBe("What_ _A Title__ _test_.md") - }) - - it("handles empty title", () => { - let docs = [createBackupDoc({ id: "doc1", title: "" })] - let locations = computeDocLocations(docs) - - expect(locations.get("doc1")?.filename).toBe("untitled.md") - }) + expect(hash1).toBe(hash2) + expect(hash1).toHaveLength(16) }) -}) - -// ============================================================================= -// Content Transformation -// ============================================================================= - -describe("transformContentForBackup", () => { - it("transforms asset: references to local paths", () => { - let assetFiles = new Map([ - ["asset123", "screenshot.png"], - ["asset456", "diagram.jpg"], - ]) - let content = `# My Doc - -Here's an image: ![Screenshot](asset:asset123) -And another: ![Diagram](asset:asset456) -` - let result = transformContentForBackup(content, assetFiles) + it("returns different hash for different content", async () => { + let hash1 = await hashContent("Content A") + let hash2 = await hashContent("Content B") - expect(result).toBe(`# My Doc - -Here's an image: ![Screenshot](assets/screenshot.png) - -And another: ![Diagram](assets/diagram.jpg) -`) + expect(hash1).not.toBe(hash2) }) - it("preserves non-asset references", () => { - let content = `![External](https://example.com/img.png) -![Local](./local/image.png) -![Asset](asset:abc123) -` - let assetFiles = new Map([["abc123", "image.png"]]) - let result = transformContentForBackup(content, assetFiles) + it("handles empty string", async () => { + let hash = await hashContent("") - expect(result).toContain("https://example.com/img.png") - expect(result).toContain("./local/image.png") - expect(result).toContain("assets/image.png") + expect(hash).toHaveLength(16) + expect(hash).toMatch(/^[a-f0-9]+$/) }) - it("leaves unknown asset references unchanged", () => { - let content = "![Unknown](asset:unknown123)" - let assetFiles = new Map() - let result = transformContentForBackup(content, assetFiles) + it("handles unicode content", async () => { + let content = "Hello 👋 World 🌍" + let hash1 = await hashContent(content) + let hash2 = await hashContent(content) - expect(result).toBe("![Unknown](asset:unknown123)") + expect(hash1).toBe(hash2) }) - it("handles multiple references to same asset", () => { - let content = `![First](asset:abc) -![Second](asset:abc) -` - let assetFiles = new Map([["abc", "image.png"]]) - let result = transformContentForBackup(content, assetFiles) + it("changes hash when only one character changes", async () => { + let hash1 = await hashContent("abc") + let hash2 = await hashContent("abd") - expect(result).toBe(`![First](assets/image.png) -![Second](assets/image.png) -`) + expect(hash1).not.toBe(hash2) }) }) -// ============================================================================= -// Expected Structure Computation -// ============================================================================= - -describe("computeExpectedStructure", () => { - it("returns empty for docs at root without assets", () => { - let docs = [createBackupDoc({ id: "doc1", title: "Doc" })] - let locations = computeDocLocations(docs) - let structure = computeExpectedStructure(docs, locations) - - expect(structure.expectedPaths.size).toBe(0) - expect(structure.expectedFiles.get("")?.has("Doc.md")).toBe(true) - }) - - it("includes path directories for nested docs", () => { - let docs = [ - createBackupDoc({ id: "doc1", title: "Doc", path: "work/notes" }), - ] - let locations = computeDocLocations(docs) - let structure = computeExpectedStructure(docs, locations) - - expect(structure.expectedPaths.has("work")).toBe(true) - expect(structure.expectedPaths.has("work/notes")).toBe(true) - expect(structure.expectedFiles.get("work/notes")?.has("Doc.md")).toBe(true) +describe("bidirectional sync exports", () => { + it("exports hashContent for testing", () => { + expect(typeof hashContent).toBe("function") }) - it("includes doc folder and assets for docs with assets", () => { - let docs = [ - createBackupDoc({ - id: "doc1", - title: "Doc", - assets: [{ id: "a1", name: "img", blob: createBlob() }], - }), - ] - let locations = computeDocLocations(docs) - let structure = computeExpectedStructure(docs, locations) - - expect(structure.expectedPaths.has("Doc")).toBe(true) - expect(structure.expectedPaths.has("Doc/assets")).toBe(true) - expect(structure.expectedFiles.get("Doc")?.has("Doc.md")).toBe(true) - }) - - it("handles mixed structure correctly", () => { - let docs = [ - createBackupDoc({ id: "doc1", title: "Simple" }), - createBackupDoc({ - id: "doc2", - title: "With Assets", - assets: [{ id: "a1", name: "img", blob: createBlob() }], - }), - createBackupDoc({ id: "doc3", title: "Nested", path: "work" }), - createBackupDoc({ - id: "doc4", - title: "Nested Assets", - path: "work", - assets: [{ id: "a2", name: "img", blob: createBlob() }], - }), - ] - let locations = computeDocLocations(docs) - let structure = computeExpectedStructure(docs, locations) - - // Root level - expect(structure.expectedFiles.get("")?.has("Simple.md")).toBe(true) - expect(structure.expectedPaths.has("With Assets")).toBe(true) - expect(structure.expectedPaths.has("With Assets/assets")).toBe(true) - - // Work path - expect(structure.expectedPaths.has("work")).toBe(true) - expect(structure.expectedFiles.get("work")?.has("Nested.md")).toBe(true) - expect(structure.expectedPaths.has("work/Nested Assets")).toBe(true) - expect(structure.expectedPaths.has("work/Nested Assets/assets")).toBe(true) - }) -}) - -// ============================================================================= -// Integration-style tests for backup structure -// ============================================================================= - -describe("backup structure", () => { - it("produces correct structure for typical backup", () => { - let docs = [ - createBackupDoc({ - id: "doc1", - title: "Meeting Notes", - path: "work", - content: "# Meeting Notes\n\nTook some notes", - }), - createBackupDoc({ - id: "doc2", - title: "Project Plan", - path: "work", - content: "# Project\n\n![Diagram](asset:diag1)", - assets: [{ id: "diag1", name: "diagram", blob: createBlob() }], - }), - createBackupDoc({ - id: "doc3", - title: "Personal Journal", - content: "# Journal\n\nToday was good", - }), - ] - - let locations = computeDocLocations(docs) - let structure = computeExpectedStructure(docs, locations) - - // Check locations - expect(locations.get("doc1")).toMatchObject({ - dirPath: "work", - filename: "Meeting Notes.md", - hasOwnFolder: false, - }) - expect(locations.get("doc2")).toMatchObject({ - dirPath: "work/Project Plan", - filename: "Project Plan.md", - hasOwnFolder: true, - }) - expect(locations.get("doc3")).toMatchObject({ - dirPath: "", - filename: "Personal Journal.md", - hasOwnFolder: false, - }) - - // Check structure - expect(structure.expectedPaths.has("work")).toBe(true) - expect(structure.expectedPaths.has("work/Project Plan")).toBe(true) - expect(structure.expectedPaths.has("work/Project Plan/assets")).toBe(true) - expect(structure.expectedFiles.get("")?.has("Personal Journal.md")).toBe( - true, - ) - expect(structure.expectedFiles.get("work")?.has("Meeting Notes.md")).toBe( - true, - ) - expect( - structure.expectedFiles.get("work/Project Plan")?.has("Project Plan.md"), - ).toBe(true) - - // Check content transformation - let doc2Loc = locations.get("doc2")! - let transformed = transformContentForBackup( - docs[1].content, - doc2Loc.assetFiles, - ) - expect(transformed).toContain("![Diagram](assets/diagram.png)") + it("exports syncFromBackup for integration tests", () => { + expect(typeof syncFromBackup).toBe("function") }) }) diff --git a/src/lib/backup.tsx b/src/lib/backup.tsx deleted file mode 100644 index 0aa5dbc..0000000 --- a/src/lib/backup.tsx +++ /dev/null @@ -1,821 +0,0 @@ -import { useState, useEffect, useRef } from "react" -import { create } from "zustand" -import { persist } from "zustand/middleware" -import { useAccount, useCoState } from "jazz-tools/react" -import { co, type ResolveQuery } from "jazz-tools" -import { FolderOpen, AlertCircle } from "lucide-react" -import { UserAccount, Document, Space } from "@/schema" -import { getDocumentTitle } from "@/lib/document-utils" -import { getPath } from "@/editor/frontmatter" -import { Button } from "@/components/ui/button" -import { get as idbGet, set as idbSet, del as idbDel } from "idb-keyval" -import { - computeDocLocations, - transformContentForBackup, - computeExpectedStructure, - type BackupDoc, -} from "@/lib/backup-sync" - -export { - BackupSubscriber, - SpacesBackupSubscriber, - BackupSettings, - SpaceBackupSettings, - useSpaceBackupPath, - getSpaceBackupPath, - setSpaceBackupPath, - clearSpaceBackupPath, - enableBackup, - disableBackup, - changeBackupDirectory, - checkBackupPermission, -} - -// File System Access API type augmentation -declare global { - interface Window { - showDirectoryPicker(options?: { - mode?: "read" | "readwrite" - }): Promise - } - interface FileSystemDirectoryHandle { - entries(): AsyncIterableIterator<[string, FileSystemHandle]> - queryPermission(options: { - mode: "read" | "readwrite" - }): Promise<"granted" | "denied" | "prompt"> - requestPermission(options: { - mode: "read" | "readwrite" - }): Promise<"granted" | "denied" | "prompt"> - } -} - -let BACKUP_DEBOUNCE_MS = 5000 -let HANDLE_STORAGE_KEY = "backup-directory-handle" - -interface BackupState { - enabled: boolean - directoryName: string | null - lastBackupAt: string | null - lastError: string | null - setEnabled: (enabled: boolean) => void - setDirectoryName: (name: string | null) => void - setLastBackupAt: (date: string | null) => void - setLastError: (error: string | null) => void - reset: () => void -} - -let useBackupStore = create()( - persist( - set => ({ - enabled: false, - directoryName: null, - lastBackupAt: null, - lastError: null, - setEnabled: enabled => set({ enabled }), - setDirectoryName: directoryName => set({ directoryName }), - setLastBackupAt: lastBackupAt => set({ lastBackupAt }), - setLastError: lastError => set({ lastError }), - reset: () => - set({ - enabled: false, - directoryName: null, - lastBackupAt: null, - lastError: null, - }), - }), - { name: "backup-settings" }, - ), -) - -type LoadedDocument = co.loaded< - typeof Document, - { content: true; assets: { $each: { image: true; video: true } } } -> - -let backupQuery = { - root: { - documents: { - $each: { content: true, assets: { $each: { image: true, video: true } } }, - $onError: "catch", - }, - }, -} as const satisfies ResolveQuery - -function BackupSubscriber() { - let { enabled, setLastBackupAt, setLastError, setEnabled, setDirectoryName } = - useBackupStore() - let me = useAccount(UserAccount, { resolve: backupQuery }) - let debounceRef = useRef | null>(null) - let lastContentHashRef = useRef("") - - useEffect(() => { - if (!enabled || !me.$isLoaded) return - - let docs = me.root?.documents - if (!docs?.$isLoaded) return - - // Compute content hash to detect changes - let activeDocs = [...docs].filter(d => d?.$isLoaded && !d.deletedAt) - let contentHash = activeDocs - .map(d => `${d.$jazz.id}:${d.updatedAt?.getTime()}`) - .sort() - .join("|") - - if (contentHash === lastContentHashRef.current) return - lastContentHashRef.current = contentHash - - // Debounce backup - if (debounceRef.current) clearTimeout(debounceRef.current) - debounceRef.current = setTimeout(async () => { - try { - let handle = await getBackupHandle() - if (!handle) { - setEnabled(false) - setDirectoryName(null) - setLastError("Permission lost - please re-enable backup") - return - } - - let loadedDocs = activeDocs.filter( - (d): d is LoadedDocument => d?.$isLoaded === true, - ) - let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) - await syncBackup(handle, backupDocs) - - setLastBackupAt(new Date().toISOString()) - setLastError(null) - } catch (e) { - setLastError(e instanceof Error ? e.message : "Backup failed") - } - }, BACKUP_DEBOUNCE_MS) - - return () => { - if (debounceRef.current) clearTimeout(debounceRef.current) - } - }, [enabled, me, setLastBackupAt, setLastError, setEnabled, setDirectoryName]) - - return null -} - -// Export for settings UI -async function enableBackup(): Promise<{ - success: boolean - directoryName?: string - error?: string -}> { - let handle = await requestBackupDirectory() - if (!handle) return { success: false, error: "Cancelled" } - - useBackupStore.getState().setEnabled(true) - useBackupStore.getState().setDirectoryName(handle.name) - useBackupStore.getState().setLastError(null) - - return { success: true, directoryName: handle.name } -} - -async function disableBackup(): Promise { - await clearHandle() - useBackupStore.getState().reset() -} - -async function changeBackupDirectory(): Promise<{ - success: boolean - directoryName?: string - error?: string -}> { - let handle = await requestBackupDirectory() - if (!handle) return { success: false, error: "Cancelled" } - - useBackupStore.getState().setDirectoryName(handle.name) - useBackupStore.getState().setLastError(null) - // Trigger immediate backup by clearing the hash - return { success: true, directoryName: handle.name } -} - -async function checkBackupPermission(): Promise { - let handle = await getBackupHandle() - return handle !== null -} - -// Settings UI component - -function BackupSettings() { - let { enabled, directoryName, lastBackupAt, lastError } = useBackupStore() - let [isLoading, setIsLoading] = useState(false) - - if (!isBackupSupported()) return null - - async function handleEnable() { - setIsLoading(true) - await enableBackup() - setIsLoading(false) - } - - async function handleDisable() { - setIsLoading(true) - await disableBackup() - setIsLoading(false) - } - - async function handleChangeDirectory() { - setIsLoading(true) - await changeBackupDirectory() - setIsLoading(false) - } - - let lastBackupDate = lastBackupAt ? new Date(lastBackupAt) : null - let formattedLastBackup = lastBackupDate - ? lastBackupDate.toLocaleString() - : null - - return ( -
-

- Local Backup -

-
- {enabled ? ( - <> -
- - Backing up to folder -
-

- Folder: {directoryName} -

- {formattedLastBackup && ( -

- Last backup: {formattedLastBackup} -

- )} - {lastError && ( -
- - {lastError} -
- )} -
- - -
- - ) : ( - <> -
- Automatic backup disabled -
-

- Automatically back up your documents to a folder on this device. -

- - - )} -
-
- ) -} - -// Space-specific backup path settings stored in localStorage - -let SPACE_BACKUP_KEY_PREFIX = "backup-settings-space-" - -interface SpaceBackupState { - directoryName: string | null -} - -function getSpaceBackupPath(spaceId: string): string | null { - try { - let key = getSpaceBackupStorageKey(spaceId) - let stored = localStorage.getItem(key) - if (!stored) return null - let parsed = JSON.parse(stored) as SpaceBackupState - return parsed.directoryName - } catch { - return null - } -} - -function setSpaceBackupPath(spaceId: string, directoryName: string): void { - let key = getSpaceBackupStorageKey(spaceId) - let state: SpaceBackupState = { directoryName } - localStorage.setItem(key, JSON.stringify(state)) -} - -function clearSpaceBackupPath(spaceId: string): void { - let key = getSpaceBackupStorageKey(spaceId) - localStorage.removeItem(key) -} - -function useSpaceBackupPath(spaceId: string): { - directoryName: string | null - setDirectoryName: (name: string | null) => void -} { - let [directoryName, setDirectoryNameState] = useState(() => - getSpaceBackupPath(spaceId), - ) - - function setDirectoryName(name: string | null) { - if (name) { - setSpaceBackupPath(spaceId, name) - } else { - clearSpaceBackupPath(spaceId) - } - setDirectoryNameState(name) - } - - return { directoryName, setDirectoryName } -} - -interface SpaceBackupSettingsProps { - spaceId: string - isAdmin: boolean -} - -function SpaceBackupSettings({ spaceId, isAdmin }: SpaceBackupSettingsProps) { - let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) - let [isLoading, setIsLoading] = useState(false) - - if (!isBackupSupported()) return null - - async function handleChooseFolder() { - setIsLoading(true) - try { - let handle = await window.showDirectoryPicker({ mode: "readwrite" }) - // Store handle in IndexedDB with space-specific key - await idbSet(`${HANDLE_STORAGE_KEY}-space-${spaceId}`, handle) - setDirectoryName(handle.name) - } catch (e) { - if (!(e instanceof Error && e.name === "AbortError")) { - console.error("Failed to select folder:", e) - } - } finally { - setIsLoading(false) - } - } - - async function handleChangeFolder() { - await handleChooseFolder() - } - - async function handleClear() { - setIsLoading(true) - try { - await idbDel(`${HANDLE_STORAGE_KEY}-space-${spaceId}`) - setDirectoryName(null) - } finally { - setIsLoading(false) - } - } - - return ( -
-

- Local Backup -

-
- {directoryName ? ( - <> -
- - Backup folder set -
-

- Folder: {directoryName} -

-
- - -
- - ) : ( - <> -
- No backup folder set -
-

- Set a backup folder for this space's documents. -

- - - )} -
-
- ) -} - -// Component that subscribes to all spaces and renders backup subscribers for each - -function SpacesBackupSubscriber() { - let me = useAccount(UserAccount, { resolve: spacesBackupQuery }) - let [storageVersion, setStorageVersion] = useState(0) - - // Re-render on storage changes (for when user sets backup path in settings) - useEffect(() => { - function handleStorageChange(e: StorageEvent) { - if (e.key?.startsWith(SPACE_BACKUP_KEY_PREFIX)) { - setStorageVersion(v => v + 1) - } - } - - window.addEventListener("storage", handleStorageChange) - return () => window.removeEventListener("storage", handleStorageChange) - }, []) - - // Compute spaces with backup paths - recomputed when me changes or storage changes - let spacesWithBackup = getSpacesWithBackup(me, storageVersion) - - return ( - <> - {spacesWithBackup.map(spaceId => ( - - ))} - - ) -} - -// ============================================================================= -// Helper functions (used by exported functions above) -// ============================================================================= - -// Space backup subscriber - handles backup sync for a single space - -let SPACE_BACKUP_DEBOUNCE_MS = 5000 - -interface SpaceBackupSubscriberProps { - spaceId: string -} - -function SpaceBackupSubscriber({ spaceId }: SpaceBackupSubscriberProps) { - let { directoryName, setDirectoryName } = useSpaceBackupPath(spaceId) - let debounceRef = useRef | null>(null) - let lastContentHashRef = useRef("") - - // Load space with documents - let space = useCoState(Space, spaceId as Parameters[1], { - resolve: { - documents: { - $each: { content: true, assets: { $each: { image: true } } }, - $onError: "catch", - }, - }, - }) - - useEffect(() => { - // Skip if no backup folder configured - if (!directoryName) return - // Skip if space not loaded - if (!space?.$isLoaded || !space.documents?.$isLoaded) return - - let docs = space.documents - let activeDocs = [...docs].filter(d => d?.$isLoaded && !d.deletedAt) - - // Compute content hash to detect changes - let contentHash = activeDocs - .map(d => `${d.$jazz.id}:${d.updatedAt?.getTime()}`) - .sort() - .join("|") - - if (contentHash === lastContentHashRef.current) return - lastContentHashRef.current = contentHash - - // Debounce backup - if (debounceRef.current) clearTimeout(debounceRef.current) - debounceRef.current = setTimeout(async () => { - try { - let handle = await getSpaceBackupHandle(spaceId) - if (!handle) { - // Permission lost, clear the setting - setDirectoryName(null) - return - } - - let loadedDocs = activeDocs.filter( - (d): d is LoadedDocument => d?.$isLoaded === true, - ) - let backupDocs = await Promise.all(loadedDocs.map(prepareBackupDoc)) - await syncBackup(handle, backupDocs) - } catch (e) { - console.error(`Space backup failed for ${spaceId}:`, e) - } - }, SPACE_BACKUP_DEBOUNCE_MS) - - return () => { - if (debounceRef.current) clearTimeout(debounceRef.current) - } - }, [directoryName, space, spaceId, setDirectoryName]) - - return null -} - -let spacesBackupQuery = { - root: { - spaces: true, - }, -} as const satisfies ResolveQuery - -function isBackupSupported(): boolean { - return "showDirectoryPicker" in window -} - -async function getStoredHandle(): Promise { - try { - let handle = await idbGet(HANDLE_STORAGE_KEY) - return handle ?? null - } catch { - return null - } -} - -async function storeHandle(handle: FileSystemDirectoryHandle): Promise { - await idbSet(HANDLE_STORAGE_KEY, handle) -} - -async function clearHandle(): Promise { - await idbDel(HANDLE_STORAGE_KEY) -} - -async function verifyPermission( - handle: FileSystemDirectoryHandle, -): Promise { - let opts = { mode: "readwrite" } as const - if ((await handle.queryPermission(opts)) === "granted") return true - if ((await handle.requestPermission(opts)) === "granted") return true - return false -} - -async function requestBackupDirectory(): Promise { - try { - let handle = await window.showDirectoryPicker({ mode: "readwrite" }) - await storeHandle(handle) - return handle - } catch (e) { - if (e instanceof Error && e.name === "AbortError") return null - throw e - } -} - -async function getBackupHandle(): Promise { - let handle = await getStoredHandle() - if (!handle) return null - let hasPermission = await verifyPermission(handle) - if (!hasPermission) return null - return handle -} - -async function prepareBackupDoc(doc: LoadedDocument): Promise { - let content = doc.content?.toString() ?? "" - let title = getDocumentTitle(doc) - let path = getPath(content) - - let assets: BackupDoc["assets"] = [] - if (doc.assets?.$isLoaded) { - for (let asset of [...doc.assets]) { - if (!asset?.$isLoaded) continue - - let blob: Blob | undefined - if (asset.type === "image" && asset.image?.$isLoaded) { - let original = asset.image.original - if (original?.$isLoaded) { - blob = original.toBlob() - } - } else if (asset.type === "video" && asset.video?.$isLoaded) { - blob = await asset.video.toBlob() - } - - if (blob) { - assets.push({ id: asset.$jazz.id, name: asset.name, blob }) - } - } - } - - return { id: doc.$jazz.id, title, content, path, assets } -} - -async function getOrCreateDirectory( - parent: FileSystemDirectoryHandle, - path: string, -): Promise { - let parts = path.split("/").filter(Boolean) - let current = parent - for (let part of parts) { - current = await current.getDirectoryHandle(part, { create: true }) - } - return current -} - -async function writeFile( - dir: FileSystemDirectoryHandle, - name: string, - content: string | Blob, -): Promise { - let file = await dir.getFileHandle(name, { create: true }) - let writable = await file.createWritable() - await writable.write(content) - await writable.close() -} - -async function deleteFile( - dir: FileSystemDirectoryHandle, - name: string, -): Promise { - try { - await dir.removeEntry(name) - } catch { - // File doesn't exist, ignore - } -} - -async function listFiles(dir: FileSystemDirectoryHandle): Promise { - let files: string[] = [] - for await (let [name, handle] of dir.entries()) { - if (handle.kind === "file") files.push(name) - } - return files -} - -async function listDirectories( - dir: FileSystemDirectoryHandle, -): Promise { - let dirs: string[] = [] - for await (let [name, handle] of dir.entries()) { - if (handle.kind === "directory") dirs.push(name) - } - return dirs -} - -async function syncBackup( - handle: FileSystemDirectoryHandle, - docs: BackupDoc[], -): Promise { - let docLocations = computeDocLocations(docs) - - // Write all documents and their assets - for (let doc of docs) { - let loc = docLocations.get(doc.id)! - let dir = loc.dirPath - ? await getOrCreateDirectory(handle, loc.dirPath) - : handle - - let exportedContent = transformContentForBackup(doc.content, loc.assetFiles) - await writeFile(dir, loc.filename, exportedContent) - - // Write assets if any - if (doc.assets.length > 0) { - let assetsDir = await dir.getDirectoryHandle("assets", { create: true }) - for (let asset of doc.assets) { - let filename = loc.assetFiles.get(asset.id)! - await writeFile(assetsDir, filename, asset.blob) - } - } - } - - // Clean up orphaned files and directories - await cleanupOrphanedFiles(handle, docs, docLocations) -} - -async function cleanupOrphanedFiles( - handle: FileSystemDirectoryHandle, - docs: BackupDoc[], - docLocations: Map< - string, - ReturnType extends Map - ? V - : never - >, -): Promise { - let { expectedPaths, expectedFiles } = computeExpectedStructure( - docs, - docLocations, - ) - - async function cleanDir( - dir: FileSystemDirectoryHandle, - path: string, - ): Promise { - let subdirs = await listDirectories(dir) - let hasContent = false - - for (let subdir of subdirs) { - let subPath = path ? `${path}/${subdir}` : subdir - - // Skip assets folders that belong to a doc - if (subdir === "assets" && expectedPaths.has(subPath)) { - hasContent = true - continue - } - - if (expectedPaths.has(subPath)) { - let subHandle = await dir.getDirectoryHandle(subdir) - let subHasContent = await cleanDir(subHandle, subPath) - if (subHasContent) hasContent = true - } else { - // Directory not expected, remove it - try { - await dir.removeEntry(subdir, { recursive: true }) - } catch { - // Ignore errors - } - } - } - - // Clean files in this directory - let expected = expectedFiles.get(path) ?? new Set() - let files = await listFiles(dir) - for (let file of files) { - if (file.endsWith(".md")) { - if (expected.has(file)) { - hasContent = true - } else { - await deleteFile(dir, file) - } - } - } - - return hasContent - } - - await cleanDir(handle, "") -} - -function getSpaceBackupStorageKey(spaceId: string): string { - return `${SPACE_BACKUP_KEY_PREFIX}${spaceId}` -} - -async function getSpaceBackupHandle( - spaceId: string, -): Promise { - try { - let handle = await idbGet( - `${HANDLE_STORAGE_KEY}-space-${spaceId}`, - ) - if (!handle) return null - let hasPermission = await verifyPermission(handle) - if (!hasPermission) return null - return handle - } catch { - return null - } -} - -function getSpacesWithBackup( - me: ReturnType< - typeof useAccount - >, - _storageVersion: number, -): string[] { - if (!me.$isLoaded || !me.root?.spaces?.$isLoaded) return [] - - let spaceIds: string[] = [] - for (let space of Array.from(me.root.spaces)) { - if (!space?.$isLoaded) continue - let backupPath = getSpaceBackupPath(space.$jazz.id) - if (backupPath) { - spaceIds.push(space.$jazz.id) - } - } - return spaceIds -} diff --git a/src/lib/editor-utils.ts b/src/lib/editor-utils.ts index 78d4074..1b4e9c6 100644 --- a/src/lib/editor-utils.ts +++ b/src/lib/editor-utils.ts @@ -291,7 +291,7 @@ function setupKeyboardShortcuts(opts: { } function showAutosaveToast() { - toast("Alkalyte saves automatically", { + toast("Alkalye saves automatically", { description: "Changes are saved locally and synced to the cloud while you type.", action: { diff --git a/src/lib/pwa.tsx b/src/lib/pwa.tsx index 95a81d1..d5b9415 100644 --- a/src/lib/pwa.tsx +++ b/src/lib/pwa.tsx @@ -153,7 +153,7 @@ function PWAInstallHint() { toast(
-
Install Alkalyte
+
Install Alkalye
Add to your homescreen for the best experience.
@@ -213,11 +213,11 @@ function PWAInstallDialog({ - Install Alkalyte + Install Alkalye {isMobileDevice() - ? "Add Alkalyte to your homescreen for instant access and the best experience." - : "Install Alkalyte as an app for quick access and a better experience."} + ? "Add Alkalye to your homescreen for instant access and the best experience." + : "Install Alkalye as an app for quick access and a better experience."}
diff --git a/src/main.tsx b/src/main.tsx index 2c4b58b..4203f4f 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -12,7 +12,10 @@ import { useSplashDelay, } from "./components/splash-screen" import { PWAContext, usePWAProvider, PWAInstallHint } from "./lib/pwa" -import { BackupSubscriber, SpacesBackupSubscriber } from "./lib/backup" +import { + BackupSubscriber, + SpacesBackupSubscriber, +} from "./lib/backup-subscribers" import { useCleanupDeleted } from "./lib/use-cleanup-deleted" import { init } from "@plausible-analytics/tracker" diff --git a/src/routes/settings.tsx b/src/routes/settings.tsx index 790633f..80f10ac 100644 --- a/src/routes/settings.tsx +++ b/src/routes/settings.tsx @@ -59,7 +59,7 @@ import { import { wordlist } from "@/lib/wordlist" import { Footer } from "@/components/footer" import { usePWA, useIsPWAInstalled, PWAInstallDialog } from "@/lib/pwa" -import { BackupSettings } from "@/lib/backup" +import { BackupSettings } from "@/lib/backup-settings-ui" import { Tooltip, TooltipTrigger, @@ -1046,7 +1046,7 @@ function InstallationSection() { Not installed

- Install Alkalyte to your device for the best experience. + Install Alkalye to your device for the best experience.