diff --git a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift index 87aec5bd..2e9378c8 100644 --- a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift +++ b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift @@ -89,7 +89,7 @@ public class FilesDatabaseManager { } }, - objectTypes: [ItemMetadata.self] + objectTypes: [ItemMetadata.self, RemoteFileChunk.self] ) self.init(realmConfig: config) } @@ -340,6 +340,9 @@ public class FilesDatabaseManager { result.downloaded = false } else if result.isUpload { result.uploaded = false + result.chunkUploadId = UUID().uuidString + } else if status == .normal, metadata.isUpload { + result.chunkUploadId = "" } Self.logger.debug( diff --git a/Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift b/Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift index 97a9e041..990a9e5f 100644 --- a/Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift +++ b/Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift @@ -8,8 +8,17 @@ import Realm import RealmSwift +extension Results where Element: RemoteFileChunk { + func toUnmanagedResults() -> [RemoteFileChunk] { + return map { RemoteFileChunk(value: $0) } + } +} + extension Results where Element: ItemMetadata { func toUnmanagedResults() -> [ItemMetadata] { return map { ItemMetadata(value: $0) } } } + + + diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 8fa20c3c..428bee5e 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -80,6 +80,130 @@ extension NextcloudKit: RemoteInterface { } } + public func chunkedUpload( + localPath: String, + remotePath: String, + remoteChunkStoreFolderName: String = UUID().uuidString, + chunkSize: Int, + remainingChunks: [RemoteFileChunk], + creationDate: Date? = nil, + modificationDate: Date? = nil, + account: Account, + options: NKRequestOptions = .init(), + currentNumChunksUpdateHandler: @escaping (_ num: Int) -> Void = { _ in }, + chunkCounter: @escaping (_ counter: Int) -> Void = { _ in }, + chunkUploadStartHandler: @escaping (_ filesChunk: [RemoteFileChunk]) -> Void = { _ in }, + requestHandler: @escaping (_ request: UploadRequest) -> Void = { _ in }, + taskHandler: @escaping (_ task: URLSessionTask) -> Void = { _ in }, + progressHandler: @escaping (Progress) -> Void = { _ in }, + chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void = { _ in } + ) async -> ( + account: String, + fileChunks: [RemoteFileChunk]?, + file: NKFile?, + afError: AFError?, + remoteError: NKError + ) { + guard let remoteUrl = URL(string: remotePath) else { + uploadLogger.error("NCKit ext: Could not get url from \(remotePath, privacy: .public)") + return ("", nil, nil, nil, .urlError) + } + let localUrl = URL(fileURLWithPath: localPath) + + let fm = FileManager.default + let chunksOutputDirectoryUrl = + fm.temporaryDirectory.appendingPathComponent(remoteChunkStoreFolderName) + do { + try fm.createDirectory(at: chunksOutputDirectoryUrl, withIntermediateDirectories: true) + } catch let error { + uploadLogger.error( + """ + Could not create temporary directory for chunked files: \(error, privacy: .public) + """ + ) + return ("", nil, nil, nil, .urlError) + } + + var directory = localUrl.deletingLastPathComponent().path + if directory.last == "/" { + directory.removeLast() + } + let fileChunksOutputDirectory = chunksOutputDirectoryUrl.path + let fileName = localUrl.lastPathComponent + let destinationFileName = remoteUrl.lastPathComponent + guard let serverUrl = remoteUrl + .deletingLastPathComponent() + .absoluteString + .removingPercentEncoding + else { + uploadLogger.error( + "NCKit ext: Could not get server url from \(remotePath, privacy: .public)" + ) + return ("", nil, nil, nil, .urlError) + } + let fileChunks = remainingChunks.toNcKitChunks() + + uploadLogger.info( + """ + Beginning chunked upload of: \(localPath, privacy: .public) + directory: \(directory, privacy: .public) + fileChunksOutputDirectory: \(fileChunksOutputDirectory, privacy: .public) + fileName: \(fileName, privacy: .public) + destinationFileName: \(destinationFileName, privacy: .public) + date: \(modificationDate?.debugDescription ?? "", privacy: .public) + creationDate: \(creationDate?.debugDescription ?? "", privacy: .public) + serverUrl: \(serverUrl, privacy: .public) + chunkFolder: \(remoteChunkStoreFolderName, privacy: .public) + filesChunk: \(fileChunks, privacy: .public) + chunkSize: \(chunkSize, privacy: .public) + """ + ) + + return await withCheckedContinuation { continuation in + uploadChunk( + directory: directory, + fileChunksOutputDirectory: fileChunksOutputDirectory, + fileName: fileName, + destinationFileName: destinationFileName, + date: modificationDate, + creationDate: creationDate, + serverUrl: serverUrl, + chunkFolder: remoteChunkStoreFolderName, + filesChunk: fileChunks, + chunkSize: chunkSize, + account: account.ncKitAccount, + options: options, + numChunks: currentNumChunksUpdateHandler, + counterChunk: chunkCounter, + start: { processedChunks in + let chunks = RemoteFileChunk.fromNcKitChunks( + processedChunks, remoteChunkStoreFolderName: remoteChunkStoreFolderName + ) + chunkUploadStartHandler(chunks) + }, + requestHandler: requestHandler, + taskHandler: taskHandler, + progressHandler: { totalBytesExpected, totalBytes, fractionCompleted in + let currentProgress = Progress(totalUnitCount: totalBytesExpected) + currentProgress.completedUnitCount = totalBytes + progressHandler(currentProgress) + }, + uploaded: { uploadedChunk in + let chunk = RemoteFileChunk( + ncKitChunk: uploadedChunk, + remoteChunkStoreFolderName: remoteChunkStoreFolderName + ) + chunkUploadCompleteHandler(chunk) + } + ) { account, receivedChunks, file, afError, error in + let chunks = RemoteFileChunk.fromNcKitChunks( + receivedChunks ?? [], remoteChunkStoreFolderName: remoteChunkStoreFolderName + ) + continuation.resume(returning: (account, chunks, file, afError, error)) + } + } + } + public func move( remotePathSource: String, remotePathDestination: String, diff --git a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift index f33d8b95..8c5d47f5 100644 --- a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift @@ -52,6 +52,31 @@ public protocol RemoteInterface { remoteError: NKError ) + func chunkedUpload( + localPath: String, + remotePath: String, + remoteChunkStoreFolderName: String, + chunkSize: Int, + remainingChunks: [RemoteFileChunk], + creationDate: Date?, + modificationDate: Date?, + account: Account, + options: NKRequestOptions, + currentNumChunksUpdateHandler: @escaping (_ num: Int) -> Void, + chunkCounter: @escaping (_ counter: Int) -> Void, + chunkUploadStartHandler: @escaping (_ filesChunk: [RemoteFileChunk]) -> Void, + requestHandler: @escaping (_ request: UploadRequest) -> Void, + taskHandler: @escaping (_ task: URLSessionTask) -> Void, + progressHandler: @escaping (Progress) -> Void, + chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void + ) async -> ( + account: String, + fileChunks: [RemoteFileChunk]?, + file: NKFile?, + afError: AFError?, + remoteError: NKError + ) + func move( remotePathSource: String, remotePathDestination: String, diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift index b6565d6f..b101812b 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift @@ -105,13 +105,17 @@ extension Item { progress: Progress, dbManager: FilesDatabaseManager ) async -> (Item?, Error?) { - let (_, ocId, etag, date, size, _, _, error) = await remoteInterface.upload( - remotePath: remotePath, - localPath: localPath, + let chunkUploadId = + itemTemplate.itemIdentifier.rawValue.replacingOccurrences(of: "/", with: "") + let (ocId, _, etag, date, size, _, error) = await upload( + fileLocatedAt: localPath, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + usingChunkUploadId: chunkUploadId, + dbManager: dbManager, creationDate: itemTemplate.creationDate as? Date, modificationDate: itemTemplate.contentModificationDate as? Date, - account: account, - options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in if let domain = domain { @@ -147,8 +151,8 @@ extension Item { filename: \(itemTemplate.filename, privacy: .public) ocId: \(ocId, privacy: .public) etag: \(etag ?? "", privacy: .public) - date: \(date ?? NSDate(), privacy: .public) - size: \(size, privacy: .public), + date: \(date ?? Date(), privacy: .public) + size: \(Int(size ?? -1), privacy: .public), account: \(account.ncKitAccount, privacy: .public) """ ) @@ -157,20 +161,20 @@ extension Item { Self.logger.warning( """ Created item upload reported as successful, but there are differences between - the received file size (\(size, privacy: .public)) + the received file size (\(Int(size ?? -1), privacy: .public)) and the original file size (\(itemTemplate.documentSize??.int64Value ?? 0)) """ ) } let newMetadata = ItemMetadata() - newMetadata.date = (date ?? NSDate()) as Date + newMetadata.date = date ?? Date() newMetadata.etag = etag ?? "" newMetadata.account = account.ncKitAccount newMetadata.fileName = itemTemplate.filename newMetadata.fileNameView = itemTemplate.filename newMetadata.ocId = ocId - newMetadata.size = size + newMetadata.size = size ?? 0 newMetadata.contentType = itemTemplate.contentType?.preferredMIMEType ?? "" newMetadata.directory = false newMetadata.serverUrl = parentItemRemotePath @@ -302,13 +306,14 @@ extension Item { Handling child bundle or package file at: \(childUrlPath, privacy: .public) """ ) - let (_, _, _, _, _, _, _, error) = await remoteInterface.upload( - remotePath: childRemoteUrl, - localPath: childUrlPath, + let (_, _, _, _, _, _, error) = await upload( + fileLocatedAt: childUrlPath, + toRemotePath: childRemoteUrl, + usingRemoteInterface: remoteInterface, + withAccount: account, + dbManager: dbManager, creationDate: childUrlAttributes.creationDate, modificationDate: childUrlAttributes.contentModificationDate, - account: account, - options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in if let domain { diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift index 576ef94a..104b9ef9 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift @@ -118,9 +118,9 @@ public extension Item { } let updatedMetadata = await withCheckedContinuation { continuation in - dbManager.setStatusForItemMetadata( - metadata, status: ItemMetadata.Status.uploading - ) { continuation.resume(returning: $0) } + dbManager.setStatusForItemMetadata(metadata, status: .uploading) { + continuation.resume(returning: $0) + } } if updatedMetadata == nil { @@ -132,13 +132,15 @@ public extension Item { ) } - let (_, _, etag, date, size, _, _, error) = await remoteInterface.upload( - remotePath: remotePath, - localPath: newContents.path, + let (_, _, etag, date, size, _, error) = await upload( + fileLocatedAt: newContents.path, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + usingChunkUploadId: metadata.chunkUploadId, + dbManager: dbManager, creationDate: newCreationDate, modificationDate: newContentModificationDate, - account: account, - options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in if let domain { @@ -180,21 +182,28 @@ public extension Item { Self.logger.warning( """ Item content modification upload reported as successful, - but there are differences between the received file size (\(size, privacy: .public)) + but there are differences between the received file size (\(size ?? -1, privacy: .public)) and the original file size (\(self.documentSize?.int64Value ?? 0)) """ ) } - let newMetadata = ItemMetadata(value: metadata) - newMetadata.date = (date ?? NSDate()) as Date + let newMetadata: ItemMetadata = await { + guard let updatedMetadata else { return ItemMetadata(value: metadata) } + return await withCheckedContinuation { continuation in + dbManager.setStatusForItemMetadata(updatedMetadata, status: .normal) { updatedMeta in + continuation.resume(returning: updatedMeta ?? ItemMetadata(value: updatedMetadata)) + } + } + }() + + newMetadata.date = date ?? Date() newMetadata.etag = etag ?? metadata.etag newMetadata.ocId = ocId - newMetadata.size = size + newMetadata.size = size ?? 0 newMetadata.session = "" newMetadata.sessionError = "" newMetadata.sessionTaskIdentifier = 0 - newMetadata.status = ItemMetadata.Status.normal.rawValue newMetadata.downloaded = true newMetadata.uploaded = true @@ -388,13 +397,14 @@ public extension Item { Handling child bundle or package file at: \(childUrlPath, privacy: .public) """ ) - let (_, _, _, _, _, _, _, error) = await remoteInterface.upload( - remotePath: childRemoteUrl, - localPath: childUrlPath, + let (_, _, _, _, _, _, error) = await upload( + fileLocatedAt: childUrlPath, + toRemotePath: childRemoteUrl, + usingRemoteInterface: remoteInterface, + withAccount: account, + dbManager: dbManager, creationDate: childUrlAttributes.creationDate, modificationDate: childUrlAttributes.contentModificationDate, - account: account, - options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in if let domain { diff --git a/Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift b/Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift index 2f414106..51dd3d94 100644 --- a/Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift +++ b/Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift @@ -44,7 +44,7 @@ public class ItemMetadata: Object { @Persisted public var account = "" @Persisted public var assetLocalIdentifier = "" @Persisted public var checksums = "" - @Persisted public var chunk: Bool = false + @Persisted public var chunkUploadId: String = "" @Persisted public var classFile = "" @Persisted public var commentsUnread: Bool = false @Persisted public var contentType = "" diff --git a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift new file mode 100644 index 00000000..c04d0b10 --- /dev/null +++ b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift @@ -0,0 +1,51 @@ +// +// RemoteFileChunk.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 2025-01-08. +// + +import Foundation +import NextcloudKit +import RealmSwift + +public class RemoteFileChunk: Object { + @Persisted public var fileName: String + @Persisted public var size: Int64 + @Persisted public var remoteChunkStoreFolderName: String + + public static func fromNcKitChunks( + _ chunks: [(fileName: String, size: Int64)], remoteChunkStoreFolderName: String + ) -> [RemoteFileChunk] { + chunks.map { + RemoteFileChunk(ncKitChunk: $0, remoteChunkStoreFolderName: remoteChunkStoreFolderName) + } + } + + public convenience init( + ncKitChunk: (fileName: String, size: Int64), remoteChunkStoreFolderName: String + ) { + self.init( + fileName: ncKitChunk.fileName, + size: ncKitChunk.size, + remoteChunkStoreFolderName: remoteChunkStoreFolderName + ) + } + + public convenience init(fileName: String, size: Int64, remoteChunkStoreFolderName: String) { + self.init() + self.fileName = fileName + self.size = size + self.remoteChunkStoreFolderName = remoteChunkStoreFolderName + } + + func toNcKitChunk() -> (fileName: String, size: Int64) { + (fileName, size) + } +} + +extension Array { + func toNcKitChunks() -> [(fileName: String, size: Int64)] { + map { ($0.fileName, $0.size) } + } +} diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift new file mode 100644 index 00000000..3fb74501 --- /dev/null +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -0,0 +1,142 @@ +// +// Upload.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 2024-12-29. +// + +import Alamofire +import Foundation +import NextcloudKit +import OSLog +import RealmSwift + +let defaultFileChunkSize = 10_000_000 // 10 MB +let uploadLogger = Logger(subsystem: Logger.subsystem, category: "upload") + +func upload( + fileLocatedAt localFilePath: String, + toRemotePath remotePath: String, + usingRemoteInterface remoteInterface: RemoteInterface, + withAccount account: Account, + inChunksSized chunkSize: Int = defaultFileChunkSize, + usingChunkUploadId chunkUploadId: String = UUID().uuidString, + dbManager: FilesDatabaseManager = .shared, + creationDate: Date? = nil, + modificationDate: Date? = nil, + options: NKRequestOptions = .init(queue: .global(qos: .utility)), + requestHandler: @escaping (UploadRequest) -> Void = { _ in }, + taskHandler: @escaping (URLSessionTask) -> Void = { _ in }, + progressHandler: @escaping (Progress) -> Void = { _ in }, + chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void = { _ in } +) async -> ( + ocId: String?, + chunks: [RemoteFileChunk]?, + etag: String?, + date: Date?, + size: Int64?, + afError: AFError?, + remoteError: NKError +) { + let fileSize = + (try? FileManager.default.attributesOfItem(atPath: localFilePath)[.size] as? Int64) ?? 0 + guard fileSize > chunkSize else { + let (_, ocId, etag, date, size, _, afError, remoteError) = await remoteInterface.upload( + remotePath: remotePath, + localPath: localFilePath, + creationDate: creationDate, + modificationDate: modificationDate, + account: account, + options: options, + requestHandler: requestHandler, + taskHandler: taskHandler, + progressHandler: progressHandler + ) + + return (ocId, nil, etag, date as? Date, size, afError, remoteError) + } + + uploadLogger.info( + """ + Performing chunked upload to \(remotePath, privacy: .public) + localFilePath: \(localFilePath, privacy: .public) + remoteChunkStoreFolderName: \(chunkUploadId, privacy: .public) + chunkSize: \(chunkSize, privacy: .public) + """ + ) + + let remainingChunks = dbManager + .ncDatabase() + .objects(RemoteFileChunk.self) + .toUnmanagedResults() + .filter { $0.remoteChunkStoreFolderName == chunkUploadId } + + let (_, chunks, file, afError, remoteError) = await remoteInterface.chunkedUpload( + localPath: localFilePath, + remotePath: remotePath, + remoteChunkStoreFolderName: chunkUploadId, + chunkSize: chunkSize, + remainingChunks: remainingChunks, + creationDate: creationDate, + modificationDate: modificationDate, + account: account, + options: options, + currentNumChunksUpdateHandler: { _ in }, + chunkCounter: { currentChunk in + uploadLogger.info( + """ + \(localFilePath, privacy: .public) current chunk: \(currentChunk, privacy: .public) + """ + ) + }, + chunkUploadStartHandler: { chunks in + uploadLogger.info("\(localFilePath, privacy: .public) chunked upload starting...") + + // Do not add chunks to database if we have done this already + guard remainingChunks.isEmpty else { return } + + let db = dbManager.ncDatabase() + do { + try db.write { db.add(chunks.map { RemoteFileChunk(value: $0) }) } + } catch let error { + uploadLogger.error( + """ + Could not write chunks to db, won't be able to resume upload if transfer stops. + \(error.localizedDescription, privacy: .public) + """ + ) + } + }, + requestHandler: requestHandler, + taskHandler: taskHandler, + progressHandler: progressHandler, + chunkUploadCompleteHandler: { chunk in + uploadLogger.info( + "\(localFilePath, privacy: .public) chunk \(chunk.fileName, privacy: .public) done" + ) + let db = dbManager.ncDatabase() + do { + try db.write { + let dbChunks = db.objects(RemoteFileChunk.self) + dbChunks + .filter { + $0.remoteChunkStoreFolderName == chunkUploadId && + $0.fileName == chunk.fileName + } + .forEach { db.delete($0) } } + } catch let error { + uploadLogger.error( + """ + Could not delete chunks in db, won't resume upload correctly if transfer stops. + \(error.localizedDescription, privacy: .public) + """ + ) + } + chunkUploadCompleteHandler(chunk) + } + ) + + uploadLogger.info("\(localFilePath, privacy: .public) successfully uploaded in chunks") + + return (file?.ocId, chunks, file?.etag, file?.date, file?.size, afError, remoteError) +} diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 9f750f9d..ca1f8d53 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -17,6 +17,8 @@ public class MockRemoteInterface: RemoteInterface { public var rootItem: MockRemoteItem? public var delegate: (any NextcloudKitDelegate)? public var rootTrashItem: MockRemoteItem? + public var currentChunks: [String: [RemoteFileChunk]] = [:] + public var completedChunkTransferSize: [String: Int64] = [:] public init(rootItem: MockRemoteItem? = nil, rootTrashItem: MockRemoteItem? = nil) { self.rootItem = rootItem @@ -53,7 +55,10 @@ public class MockRemoteInterface: RemoteInterface { } func item(remotePath: String, account: Account) -> MockRemoteItem? { - guard let rootItem, !remotePath.isEmpty else { return nil } + guard let rootItem, !remotePath.isEmpty else { + print("Invalid root item or remote path, cannot get item in item tree.") + return nil + } let sanitisedPath = sanitisedPath(remotePath, account: account) guard sanitisedPath != "/" else { @@ -222,6 +227,93 @@ public class MockRemoteInterface: RemoteInterface { ) } + public func chunkedUpload( + localPath: String, + remotePath: String, + remoteChunkStoreFolderName: String, + chunkSize: Int, + remainingChunks: [RemoteFileChunk], + creationDate: Date?, + modificationDate: Date?, + account: Account, + options: NKRequestOptions, + currentNumChunksUpdateHandler: @escaping (Int) -> Void = { _ in }, + chunkCounter: @escaping (Int) -> Void = { _ in }, + chunkUploadStartHandler: @escaping ([RemoteFileChunk]) -> Void = { _ in }, + requestHandler: @escaping (UploadRequest) -> Void = { _ in }, + taskHandler: @escaping (URLSessionTask) -> Void = { _ in }, + progressHandler: @escaping (Progress) -> Void = { _ in }, + chunkUploadCompleteHandler: @escaping (RemoteFileChunk) -> Void = { _ in } + ) async -> ( + account: String, + fileChunks: [RemoteFileChunk]?, + file: NKFile?, + afError: AFError?, + remoteError: NKError + ) { + guard let remoteUrl = URL(string: remotePath) else { + print("Invalid remote path!") + return ("", nil, nil, nil, .urlError) + } + + // Create temp directory for file and create chunks within it + let fm = FileManager.default + let tempDirectoryUrl = fm.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try! fm.createDirectory(atPath: tempDirectoryUrl.path, withIntermediateDirectories: true) + + // Access local file and gather metadata + let fileSize = try! fm.attributesOfItem(atPath: localPath)[.size] as! Int + + var remainingFileSize = fileSize + let numChunks = Int(ceil(Double(fileSize) / Double(chunkSize))) + let newChunks = !remainingChunks.isEmpty + ? remainingChunks + : (0.. MockRemoteItem { + MockRemoteItem( + identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, + versionIdentifier: "root", + name: "root", + remotePath: account.davFilesUrl, + directory: true, + account: account.ncKitAccount, + username: account.username, + userId: account.id, + serverUrl: account.serverUrl + ) + } + + public static func rootTrashItem(account: Account) -> MockRemoteItem { + return MockRemoteItem( + identifier: NSFileProviderItemIdentifier.trashContainer.rawValue, + versionIdentifier: "root", + name: "root", + remotePath: account.trashUrl, + directory: true, + account: account.ncKitAccount, + username: account.username, + userId: account.id, + serverUrl: account.serverUrl + ) + } + public init( identifier: String, versionIdentifier: String = "0", diff --git a/Tests/InterfaceTests/MockRemoteInterfaceTests.swift b/Tests/InterfaceTests/MockRemoteInterfaceTests.swift index 29ce52d6..91c6a864 100644 --- a/Tests/InterfaceTests/MockRemoteInterfaceTests.swift +++ b/Tests/InterfaceTests/MockRemoteInterfaceTests.swift @@ -13,28 +13,8 @@ final class MockRemoteInterfaceTests: XCTestCase { static let account = Account( user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: "root", - versionIdentifier: "root", - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) - lazy var rootTrashItem = MockRemoteItem( - identifier: "root", - versionIdentifier: "root", - name: "root", - remotePath: Self.account.trashUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) + lazy var rootTrashItem = MockRemoteItem.rootTrashItem(account: Self.account) override func tearDown() { rootItem.children = [] @@ -188,6 +168,122 @@ final class MockRemoteInterfaceTests: XCTestCase { // TODO: Add test for overwriting existing file } + func testChunkedUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: Self.account)) + let remotePath = Self.account.davFilesUrl + "/file.txt" + let chunkSize = 3 + var uploadedChunks = [RemoteFileChunk]() + let result = await remoteInterface.chunkedUpload( + localPath: fileUrl.path, + remotePath: remotePath, + remoteChunkStoreFolderName: UUID().uuidString, + chunkSize: chunkSize, + remainingChunks: [], + creationDate: .init(), + modificationDate: .init(), + account: Self.account, + options: .init(), + chunkUploadCompleteHandler: { uploadedChunks.append($0) } + ) + + let resultChunks = try XCTUnwrap(result.fileChunks) + let expectedChunkCount = Int(ceil(Double(data.count) / Double(chunkSize))) + + XCTAssertEqual(result.remoteError, .success) + XCTAssertEqual(resultChunks.count, expectedChunkCount) + XCTAssertNotNil(result.file) + XCTAssertEqual(result.file?.size, Int64(data.count)) + + XCTAssertEqual(uploadedChunks.count, resultChunks.count) + + let firstUploadedChunk = try XCTUnwrap(uploadedChunks.first) + let firstUploadedChunkNameInt = try XCTUnwrap(Int(firstUploadedChunk.fileName)) + let lastUploadedChunk = try XCTUnwrap(uploadedChunks.last) + let lastUploadedChunkNameInt = try XCTUnwrap(Int(lastUploadedChunk.fileName)) + XCTAssertEqual(firstUploadedChunkNameInt, 1) + XCTAssertEqual(lastUploadedChunkNameInt, expectedChunkCount) + XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) + XCTAssertEqual( + Int(lastUploadedChunk.size), data.count - ((lastUploadedChunkNameInt - 1) * chunkSize) + ) + } + + func testResumedChunkedUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let chunkSize = 3 + let uploadUuid = UUID().uuidString + let previousUploadedChunkNum = 1 + let previousUploadedChunk = RemoteFileChunk( + fileName: String(previousUploadedChunkNum), + size: Int64(chunkSize), + remoteChunkStoreFolderName: uploadUuid + ) + let previousUploadedChunks = [previousUploadedChunk] + + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: Self.account)) + remoteInterface.currentChunks = [uploadUuid: previousUploadedChunks] + + let remotePath = Self.account.davFilesUrl + "/file.txt" + + var uploadedChunks = [RemoteFileChunk]() + let result = await remoteInterface.chunkedUpload( + localPath: fileUrl.path, + remotePath: remotePath, + remoteChunkStoreFolderName: uploadUuid, + chunkSize: chunkSize, + remainingChunks: [ + RemoteFileChunk( + fileName: String(2), + size: Int64(chunkSize), + remoteChunkStoreFolderName: uploadUuid + ), + RemoteFileChunk( + fileName: String(3), + size: Int64(data.count - (chunkSize * 2)), + remoteChunkStoreFolderName: uploadUuid + ) + ], + creationDate: .init(), + modificationDate: .init(), + account: Self.account, + options: .init(), + chunkUploadCompleteHandler: { uploadedChunks.append($0) } + ) + + let resultChunks = try XCTUnwrap(result.fileChunks) + let expectedChunkCount = Int(ceil(Double(data.count) / Double(chunkSize))) + + XCTAssertEqual(result.remoteError, .success) + XCTAssertEqual(resultChunks.count, expectedChunkCount) + XCTAssertNotNil(result.file) + XCTAssertEqual(result.file?.size, Int64(data.count)) + + XCTAssertEqual(uploadedChunks.count, resultChunks.count - previousUploadedChunks.count) + + let firstUploadedChunk = try XCTUnwrap(uploadedChunks.first) + let firstUploadedChunkNameInt = try XCTUnwrap(Int(firstUploadedChunk.fileName)) + let lastUploadedChunk = try XCTUnwrap(uploadedChunks.last) + let lastUploadedChunkNameInt = try XCTUnwrap(Int(lastUploadedChunk.fileName)) + XCTAssertEqual(firstUploadedChunkNameInt, previousUploadedChunkNum + 1) + XCTAssertEqual(lastUploadedChunkNameInt, previousUploadedChunkNum + 2) + print(uploadedChunks) + XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) + XCTAssertEqual( + Int(lastUploadedChunk.size), data.count - ((lastUploadedChunkNameInt - 1) * chunkSize) + ) + } + func testMove() async { let remoteInterface = MockRemoteInterface(rootItem: rootItem) let itemA = MockRemoteItem( diff --git a/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift b/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift index 6562cb24..da32dca2 100644 --- a/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift +++ b/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift @@ -34,16 +34,7 @@ final class EnumeratorTests: XCTestCase { super.setUp() Realm.Configuration.defaultConfiguration.inMemoryIdentifier = name - rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + rootItem = MockRemoteItem.rootItem(account: Self.account) remoteFolder = MockRemoteItem( identifier: "folder", @@ -95,16 +86,7 @@ final class EnumeratorTests: XCTestCase { remoteItemB.parent = remoteFolder remoteItemC.parent = nil - rootTrashItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.trashContainer.rawValue, - name: "root", - remotePath: Self.account.trashUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + rootTrashItem = MockRemoteItem.rootTrashItem(account: Self.account) remoteTrashItemA = MockRemoteItem( identifier: "trashItemA", diff --git a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift index f5534644..a8c7e8e4 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift @@ -18,21 +18,13 @@ final class ItemCreateTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + var rootItem: MockRemoteItem! static let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) override func setUp() { super.setUp() Realm.Configuration.defaultConfiguration.inMemoryIdentifier = name + rootItem = MockRemoteItem.rootItem(account: Self.account) } override func tearDown() { @@ -356,4 +348,148 @@ final class ItemCreateTests: XCTestCase { let childrenCount = Self.dbManager.childItemCount(directoryMetadata: dbItem) XCTAssertEqual(childrenCount, 6) // Ensure all children recorded to database } + + func testCreateFileChunked() async throws { + let remoteInterface = MockRemoteInterface(rootItem: rootItem) + let fileItemMetadata = ItemMetadata() + fileItemMetadata.fileName = "file" + fileItemMetadata.fileNameView = "file" + fileItemMetadata.directory = false + fileItemMetadata.classFile = NKCommon.TypeClassFile.document.rawValue + fileItemMetadata.serverUrl = Self.account.davFilesUrl + + let tempUrl = FileManager.default.temporaryDirectory.appendingPathComponent("file") + let tempData = Data(repeating: 1, count: defaultFileChunkSize * 3) + try tempData.write(to: tempUrl) + + let fileItemTemplate = Item( + metadata: fileItemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + let (createdItemMaybe, error) = await Item.create( + basedOn: fileItemTemplate, + contents: tempUrl, + account: Self.account, + remoteInterface: remoteInterface, + progress: Progress(), + dbManager: Self.dbManager + ) + let createdItem = try XCTUnwrap(createdItemMaybe) + + XCTAssertNil(error) + XCTAssertNotNil(createdItem) + XCTAssertEqual(createdItem.metadata.fileName, fileItemMetadata.fileName) + XCTAssertEqual(createdItem.metadata.directory, fileItemMetadata.directory) + + let remoteItem = try XCTUnwrap( + rootItem.children.first { $0.identifier == createdItem.itemIdentifier.rawValue } + ) + XCTAssertEqual(remoteItem.name, fileItemMetadata.fileName) + XCTAssertEqual(remoteItem.directory, fileItemMetadata.directory) + XCTAssertEqual(remoteItem.data, tempData) + + let dbItem = try XCTUnwrap( + Self.dbManager.itemMetadata(ocId: createdItem.itemIdentifier.rawValue) + ) + XCTAssertEqual(dbItem.fileName, fileItemMetadata.fileName) + XCTAssertEqual(dbItem.fileNameView, fileItemMetadata.fileNameView) + XCTAssertEqual(dbItem.directory, fileItemMetadata.directory) + XCTAssertEqual(dbItem.serverUrl, fileItemMetadata.serverUrl) + XCTAssertEqual(dbItem.ocId, createdItem.itemIdentifier.rawValue) + } + + func testCreateFileChunkedResumed() async throws { + let expectedChunkUploadId = UUID().uuidString // Check if illegal characters are stripped + let illegalChunkUploadId = expectedChunkUploadId + "/" // Check if illegal characters are stripped + let previousUploadedChunkNum = 1 + let preexistingChunk = RemoteFileChunk( + fileName: String(previousUploadedChunkNum), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: expectedChunkUploadId + ) + + let db = Self.dbManager.ncDatabase() + try db.write { + db.add([ + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 1), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: expectedChunkUploadId + ), + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 2), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: expectedChunkUploadId + ) + ]) + } + + let remoteInterface = MockRemoteInterface(rootItem: rootItem) + remoteInterface.currentChunks = [expectedChunkUploadId: [preexistingChunk]] + + // With real new item uploads we do not have an associated ItemMetadata as the template is + // passed onto us by the OS. We cannot rely on the chunkUploadId property we usually use + // during modified item uploads. + // + // We therefore can only use the system-provided item template's itemIdentifier as the + // chunked upload identifier during new item creation. + // + // To test this situation we set the ocId of the metadata used to construct the item + // template to the chunk upload id. + let fileItemMetadata = ItemMetadata() + fileItemMetadata.ocId = illegalChunkUploadId + fileItemMetadata.fileName = "file" + fileItemMetadata.fileNameView = "file" + fileItemMetadata.directory = false + fileItemMetadata.classFile = NKCommon.TypeClassFile.document.rawValue + fileItemMetadata.serverUrl = Self.account.davFilesUrl + + let tempUrl = FileManager.default.temporaryDirectory.appendingPathComponent("file") + let tempData = Data(repeating: 1, count: defaultFileChunkSize * 3) + try tempData.write(to: tempUrl) + + let fileItemTemplate = Item( + metadata: fileItemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + let (createdItemMaybe, error) = await Item.create( + basedOn: fileItemTemplate, + contents: tempUrl, + account: Self.account, + remoteInterface: remoteInterface, + progress: Progress(), + dbManager: Self.dbManager + ) + let createdItem = try XCTUnwrap(createdItemMaybe) + + XCTAssertNil(error) + XCTAssertNotNil(createdItem) + XCTAssertEqual(createdItem.metadata.fileName, fileItemMetadata.fileName) + XCTAssertEqual(createdItem.metadata.directory, fileItemMetadata.directory) + + let remoteItem = try XCTUnwrap( + rootItem.children.first { $0.identifier == createdItem.itemIdentifier.rawValue } + ) + XCTAssertEqual(remoteItem.name, fileItemMetadata.fileName) + XCTAssertEqual(remoteItem.directory, fileItemMetadata.directory) + XCTAssertEqual(remoteItem.data, tempData) + XCTAssertEqual( + remoteInterface.completedChunkTransferSize[expectedChunkUploadId], + Int64(tempData.count) - preexistingChunk.size + ) + + let dbItem = try XCTUnwrap( + Self.dbManager.itemMetadata(ocId: createdItem.itemIdentifier.rawValue) + ) + XCTAssertEqual(dbItem.fileName, fileItemMetadata.fileName) + XCTAssertEqual(dbItem.fileNameView, fileItemMetadata.fileNameView) + XCTAssertEqual(dbItem.directory, fileItemMetadata.directory) + XCTAssertEqual(dbItem.serverUrl, fileItemMetadata.serverUrl) + XCTAssertEqual(dbItem.ocId, createdItem.itemIdentifier.rawValue) + XCTAssertTrue(dbItem.chunkUploadId.isEmpty) + } } diff --git a/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift b/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift index 0ffd786c..f4cda9b7 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift @@ -16,26 +16,8 @@ final class ItemDeleteTests: XCTestCase { static let account = Account( user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) - lazy var rootTrashItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "trash", - remotePath: Self.account.trashUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) + lazy var rootTrashItem = MockRemoteItem.rootTrashItem(account: Self.account) static let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) override func setUp() { diff --git a/Tests/NextcloudFileProviderKitTests/ItemFetchTests.swift b/Tests/NextcloudFileProviderKitTests/ItemFetchTests.swift index 2eae255c..1d1ea6bb 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemFetchTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemFetchTests.swift @@ -17,16 +17,7 @@ final class ItemFetchTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) static let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) override func setUp() { diff --git a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift index a55090d2..ad008eca 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift @@ -18,26 +18,8 @@ final class ItemModifyTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) - lazy var rootTrashItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.trashContainer.rawValue, - name: "root", - remotePath: Self.account.trashUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) + lazy var rootTrashItem = MockRemoteItem.rootTrashItem(account: Self.account) var remoteFolder: MockRemoteItem! var remoteItem: MockRemoteItem! @@ -1223,4 +1205,127 @@ final class ItemModifyTests: XCTestCase { ) XCTAssertEqual(untrashedFolderChildItem.serverUrl, remoteTrashFolder.remotePath) } + + func testModifyFileContentsChunked() async throws { + let remoteInterface = MockRemoteInterface(rootItem: rootItem) + + let itemMetadata = remoteItem.toItemMetadata(account: Self.account) + Self.dbManager.addItemMetadata(itemMetadata) + + let newContents = Data(repeating: 1, count: defaultFileChunkSize * 3) + let newContentsUrl = FileManager.default.temporaryDirectory.appendingPathComponent("test") + try newContents.write(to: newContentsUrl) + + let targetItemMetadata = ItemMetadata(value: itemMetadata) + targetItemMetadata.date = .init() + targetItemMetadata.size = Int64(newContents.count) + + let item = Item( + metadata: itemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + + let targetItem = Item( + metadata: targetItemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + targetItem.dbManager = Self.dbManager + + let (modifiedItemMaybe, error) = await item.modify( + itemTarget: targetItem, + changedFields: [.contents, .contentModificationDate], + contents: newContentsUrl, + dbManager: Self.dbManager + ) + XCTAssertNil(error) + let modifiedItem = try XCTUnwrap(modifiedItemMaybe) + + XCTAssertEqual(modifiedItem.itemIdentifier, targetItem.itemIdentifier) + XCTAssertEqual(modifiedItem.contentModificationDate, targetItem.contentModificationDate) + XCTAssertEqual(modifiedItem.documentSize?.intValue, newContents.count) + + XCTAssertEqual(remoteItem.data, newContents) + } + + func testModifyFileContentsChunkedResumed() async throws { + let chunkUploadId = UUID().uuidString + let previousUploadedChunkNum = 1 + let preexistingChunk = RemoteFileChunk( + fileName: String(previousUploadedChunkNum), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ) + + let db = Self.dbManager.ncDatabase() + try db.write { + db.add([ + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 1), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ), + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 2), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ) + ]) + } + + let remoteInterface = MockRemoteInterface(rootItem: rootItem) + remoteInterface.currentChunks = [chunkUploadId: [preexistingChunk]] + + let itemMetadata = remoteItem.toItemMetadata(account: Self.account) + itemMetadata.chunkUploadId = chunkUploadId + Self.dbManager.addItemMetadata(itemMetadata) + + let newContents = Data(repeating: 1, count: defaultFileChunkSize * 3) + let newContentsUrl = FileManager.default.temporaryDirectory.appendingPathComponent("test") + try newContents.write(to: newContentsUrl) + + let targetItemMetadata = ItemMetadata(value: itemMetadata) + targetItemMetadata.date = .init() + targetItemMetadata.size = Int64(newContents.count) + + let item = Item( + metadata: itemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + + let targetItem = Item( + metadata: targetItemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + targetItem.dbManager = Self.dbManager + + let (modifiedItemMaybe, error) = await item.modify( + itemTarget: targetItem, + changedFields: [.contents, .contentModificationDate], + contents: newContentsUrl, + dbManager: Self.dbManager + ) + XCTAssertNil(error) + let modifiedItem = try XCTUnwrap(modifiedItemMaybe) + + XCTAssertEqual(modifiedItem.itemIdentifier, targetItem.itemIdentifier) + XCTAssertEqual(modifiedItem.contentModificationDate, targetItem.contentModificationDate) + XCTAssertEqual(modifiedItem.documentSize?.intValue, newContents.count) + + XCTAssertEqual(remoteItem.data, newContents) + XCTAssertEqual( + remoteInterface.completedChunkTransferSize[chunkUploadId], + Int64(newContents.count) - preexistingChunk.size + ) + + let dbItem = try XCTUnwrap(Self.dbManager.itemMetadata(ocId: itemMetadata.ocId)) + XCTAssertTrue(dbItem.chunkUploadId.isEmpty) + } } diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift new file mode 100644 index 00000000..a5d5a499 --- /dev/null +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -0,0 +1,159 @@ +// +// UploadTests.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 2025-01-07. +// + +import RealmSwift +import TestInterface +import XCTest +@testable import NextcloudFileProviderKit + +final class UploadTests: XCTestCase { + let account = Account(user: "user", id: "id", serverUrl: "test.cloud.com", password: "1234") + let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) + + override func setUp() { + super.setUp() + Realm.Configuration.defaultConfiguration.inMemoryIdentifier = name + } + + func testStandardUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) + let remotePath = account.davFilesUrl + "/file.txt" + let result = await NextcloudFileProviderKit.upload( + fileLocatedAt: fileUrl.path, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + dbManager: dbManager + ) + + XCTAssertEqual(result.remoteError, .success) + XCTAssertNil(result.chunks) + XCTAssertEqual(result.size, Int64(data.count)) + XCTAssertNotNil(result.ocId) + XCTAssertNotNil(result.etag) + } + + func testChunkedUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) + let remotePath = account.davFilesUrl + "/file.txt" + let chunkSize = 3 + var uploadedChunks = [RemoteFileChunk]() + let result = await NextcloudFileProviderKit.upload( + fileLocatedAt: fileUrl.path, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + inChunksSized: chunkSize, + dbManager: dbManager, + chunkUploadCompleteHandler: { uploadedChunks.append($0) } + ) + let resultChunks = try XCTUnwrap(result.chunks) + let expectedChunkCount = Int(ceil(Double(data.count) / Double(chunkSize))) + + XCTAssertEqual(result.remoteError, .success) + XCTAssertEqual(resultChunks.count, expectedChunkCount) + XCTAssertEqual(result.size, Int64(data.count)) + XCTAssertNotNil(result.ocId) + XCTAssertNotNil(result.etag) + + XCTAssertEqual(uploadedChunks.count, resultChunks.count) + + let firstUploadedChunk = try XCTUnwrap(uploadedChunks.first) + let firstUploadedChunkNameInt = try XCTUnwrap(Int(firstUploadedChunk.fileName)) + let lastUploadedChunk = try XCTUnwrap(uploadedChunks.last) + let lastUploadedChunkNameInt = try XCTUnwrap(Int(lastUploadedChunk.fileName)) + XCTAssertEqual(firstUploadedChunkNameInt, 1) + XCTAssertEqual(lastUploadedChunkNameInt, expectedChunkCount) + XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) + XCTAssertEqual( + Int(lastUploadedChunk.size), data.count - ((lastUploadedChunkNameInt - 1) * chunkSize) + ) + } + + func testResumingInterruptedChunkedUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) + let chunkSize = 3 + let uploadUuid = UUID().uuidString + let previousUploadedChunkNum = 1 + let previousUploadedChunk = RemoteFileChunk( + fileName: String(previousUploadedChunkNum), + size: Int64(chunkSize), + remoteChunkStoreFolderName: uploadUuid + ) + let previousUploadedChunks = [previousUploadedChunk] + remoteInterface.currentChunks = [uploadUuid: previousUploadedChunks] + + let db = dbManager.ncDatabase() + try db.write { + db.add([ + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 1), + size: Int64(chunkSize), + remoteChunkStoreFolderName: uploadUuid + ), + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 2), + size: Int64(data.count - (chunkSize * (previousUploadedChunkNum + 1))), + remoteChunkStoreFolderName: uploadUuid + ) + ]) + } + + let remotePath = account.davFilesUrl + "/file.txt" + var uploadedChunks = [RemoteFileChunk]() + let result = await NextcloudFileProviderKit.upload( + fileLocatedAt: fileUrl.path, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + inChunksSized: chunkSize, + usingChunkUploadId: uploadUuid, + dbManager: dbManager, + chunkUploadCompleteHandler: { uploadedChunks.append($0) } + ) + let resultChunks = try XCTUnwrap(result.chunks) + let expectedChunkCount = Int(ceil(Double(data.count) / Double(chunkSize))) + + XCTAssertEqual(result.remoteError, .success) + XCTAssertEqual(resultChunks.count, expectedChunkCount) + XCTAssertEqual(result.size, Int64(data.count)) + XCTAssertNotNil(result.ocId) + XCTAssertNotNil(result.etag) + + XCTAssertEqual(uploadedChunks.count, resultChunks.count - previousUploadedChunks.count) + + let firstUploadedChunk = try XCTUnwrap(uploadedChunks.first) + let firstUploadedChunkNameInt = try XCTUnwrap(Int(firstUploadedChunk.fileName)) + let lastUploadedChunk = try XCTUnwrap(uploadedChunks.last) + let lastUploadedChunkNameInt = try XCTUnwrap(Int(lastUploadedChunk.fileName)) + XCTAssertEqual(firstUploadedChunkNameInt, previousUploadedChunkNum + 1) + XCTAssertEqual(lastUploadedChunkNameInt, previousUploadedChunkNum + 2) + print(uploadedChunks) + XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) + XCTAssertEqual( + Int(lastUploadedChunk.size), data.count - ((lastUploadedChunkNameInt - 1) * chunkSize) + ) + } +}