From 23c9dfd9a90a5b92c9eee35ee35bc5254612a2d5 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Sun, 29 Dec 2024 01:57:53 +0800 Subject: [PATCH 01/86] Add RemoteFileChunk type Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Interface/RemoteInterface.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift index f33d8b95..7f42413a 100644 --- a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift @@ -20,6 +20,8 @@ public enum AuthenticationAttemptResultState: Int { case authenticationError, connectionError, success } +public typealias RemoteFileChunk = (fileName: String, size: Int64) + public protocol RemoteInterface { func setDelegate(_ delegate: NextcloudKitDelegate) From b650cec22486b9d4b71e317155d4c2d4fe558467 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Sun, 29 Dec 2024 01:58:12 +0800 Subject: [PATCH 02/86] Add chunkedUpload method to RemoteInterface protocol Signed-off-by: Claudio Cambra --- .../Interface/RemoteInterface.swift | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift index 7f42413a..9f1449e7 100644 --- a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift @@ -54,6 +54,34 @@ public protocol RemoteInterface { remoteError: NKError ) + func chunkedUpload( + localDirectoryPath: String, + localFileName: String, + remoteParentDirectoryPath: String, + remoteChunkStoreFolderName: String, + chunkSize: Int, + remainingChunks: [RemoteFileChunk], + creationDate: Date?, + modificationDate: Date?, + account: Account, + options: NKRequestOptions, + currentNumChunksUpdateHandler: @escaping (_ num: Int) -> Void, + chunkCounter: @escaping (_ counter: Int) -> Void, + chunkUploadStartHandler: @escaping (_ filesChunk: [RemoteFileChunk]) -> Void, + requestHandler: @escaping (_ request: UploadRequest) -> Void, + taskHandler: @escaping (_ task: URLSessionTask) -> Void, + progressHandler: @escaping ( + _ totalBytesExpected: Int64, _ totalBytes: Int64, _ fractionCompleted: Double + ) -> Void, + chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void + ) async -> ( + account: String, + fileChunks: [RemoteFileChunk]?, + file: NKFile?, + afError: AFError?, + remoteError: NKError + ) + func move( remotePathSource: String, remotePathDestination: String, From ebff7dbff2b7f85f1ace3186230012cb352bda5a Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Sun, 29 Dec 2024 01:58:27 +0800 Subject: [PATCH 03/86] Implement chunkedUpload in nextcloudkit extension Signed-off-by: Claudio Cambra --- .../NextcloudKit+RemoteInterface.swift | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 8fa20c3c..3edcf8d0 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -80,6 +80,59 @@ extension NextcloudKit: RemoteInterface { } } + public func chunkedUpload( + localDirectoryPath: String, + localFileName: String, + remoteParentDirectoryPath: String, + remoteChunkStoreFolderName: String = UUID().uuidString, + chunkSize: Int, + remainingChunks: [RemoteFileChunk], + creationDate: Date? = nil, + modificationDate: Date? = nil, + account: Account, + options: NKRequestOptions = .init(), + currentNumChunksUpdateHandler: @escaping (_ num: Int) -> Void = { _ in }, + chunkCounter: @escaping (_ counter: Int) -> Void = { _ in }, + chunkUploadStartHandler: @escaping (_ filesChunk: [RemoteFileChunk]) -> Void = { _ in }, + requestHandler: @escaping (_ request: UploadRequest) -> Void = { _ in }, + taskHandler: @escaping (_ task: URLSessionTask) -> Void = { _ in }, + progressHandler: @escaping ( + _ totalBytesExpected: Int64, _ totalBytes: Int64, _ fractionCompleted: Double + ) -> Void = { _, _, _ in }, + chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void = { _ in } + ) async -> ( + account: String, + fileChunks: [RemoteFileChunk]?, + file: NKFile?, + afError: AFError?, + remoteError: NKError + ) { + return await withCheckedContinuation { continuation in + uploadChunk( + directory: localDirectoryPath, + fileName: localFileName, + date: modificationDate, + creationDate: creationDate, + serverUrl: remoteParentDirectoryPath, + chunkFolder: remoteChunkStoreFolderName, + filesChunk: remainingChunks, + chunkSize: chunkSize, + account: account.ncKitAccount, + options: options, + numChunks: currentNumChunksUpdateHandler, + counterChunk: chunkCounter, + start: chunkUploadStartHandler, + requestHandler: requestHandler, + taskHandler: taskHandler, + progressHandler: progressHandler, + uploaded: chunkUploadCompleteHandler + ) { account, filesChunk, file, afError, error in + let chunks = filesChunk as [RemoteFileChunk]? + continuation.resume(returning: (account, chunks, file, afError, error)) + } + } + } + public func move( remotePathSource: String, remotePathDestination: String, From 9640f5898882723b94f0f274740edbd2905b680c Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 16:55:56 +0900 Subject: [PATCH 04/86] Make the progressHandler for chunked upload match other methods Signed-off-by: Claudio Cambra --- .../Interface/NextcloudKit+RemoteInterface.swift | 10 ++++++---- .../Interface/RemoteInterface.swift | 4 +--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 3edcf8d0..3cde6b48 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -96,9 +96,7 @@ extension NextcloudKit: RemoteInterface { chunkUploadStartHandler: @escaping (_ filesChunk: [RemoteFileChunk]) -> Void = { _ in }, requestHandler: @escaping (_ request: UploadRequest) -> Void = { _ in }, taskHandler: @escaping (_ task: URLSessionTask) -> Void = { _ in }, - progressHandler: @escaping ( - _ totalBytesExpected: Int64, _ totalBytes: Int64, _ fractionCompleted: Double - ) -> Void = { _, _, _ in }, + progressHandler: @escaping (Progress) -> Void = { _ in }, chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void = { _ in } ) async -> ( account: String, @@ -124,7 +122,11 @@ extension NextcloudKit: RemoteInterface { start: chunkUploadStartHandler, requestHandler: requestHandler, taskHandler: taskHandler, - progressHandler: progressHandler, + progressHandler: { totalBytesExpected, totalBytes, fractionCompleted in + let currentProgress = Progress(totalUnitCount: totalBytesExpected) + currentProgress.completedUnitCount = totalBytes + progressHandler(currentProgress) + }, uploaded: chunkUploadCompleteHandler ) { account, filesChunk, file, afError, error in let chunks = filesChunk as [RemoteFileChunk]? diff --git a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift index 9f1449e7..6811d5c8 100644 --- a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift @@ -70,9 +70,7 @@ public protocol RemoteInterface { chunkUploadStartHandler: @escaping (_ filesChunk: [RemoteFileChunk]) -> Void, requestHandler: @escaping (_ request: UploadRequest) -> Void, taskHandler: @escaping (_ task: URLSessionTask) -> Void, - progressHandler: @escaping ( - _ totalBytesExpected: Int64, _ totalBytes: Int64, _ fractionCompleted: Double - ) -> Void, + progressHandler: @escaping (Progress) -> Void, chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void ) async -> ( account: String, From 9ea7e3d1a3b7d51458595629091dc2e03bceb35b Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:20:06 +0900 Subject: [PATCH 05/86] Add simple starter test for upload result struct Signed-off-by: Claudio Cambra --- .../UploadTests.swift | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 Tests/NextcloudFileProviderKitTests/UploadTests.swift diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift new file mode 100644 index 00000000..82d752ce --- /dev/null +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -0,0 +1,25 @@ +// +// UploadTests.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 2025-01-07. +// + +import XCTest +@testable import NextcloudFileProviderKit + +final class UploadTests: XCTestCase { + + func testSucceededUploadResult() { + let uploadResult = UploadResult( + ocId: nil, + chunks: nil, + etag: nil, + date: nil, + size: nil, + afError: nil, + remoteError: .success + ) + XCTAssertTrue(uploadResult.succeeded) + } +} From 9f29aed9dbfa7f3f6c483e845e12f6fd90ca6a20 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:23:29 +0900 Subject: [PATCH 06/86] Add basic chunked upload implementation for mock remote interface Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 9f750f9d..bec38c73 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -17,6 +17,7 @@ public class MockRemoteInterface: RemoteInterface { public var rootItem: MockRemoteItem? public var delegate: (any NextcloudKitDelegate)? public var rootTrashItem: MockRemoteItem? + public var currentChunks: [String: [RemoteFileChunk]] = [:] public init(rootItem: MockRemoteItem? = nil, rootTrashItem: MockRemoteItem? = nil) { self.rootItem = rootItem @@ -222,6 +223,76 @@ public class MockRemoteInterface: RemoteInterface { ) } + public func chunkedUpload( + localDirectoryPath: String, + localFileName: String, + remoteParentDirectoryPath: String, + remoteChunkStoreFolderName: String, + chunkSize: Int, + remainingChunks: [RemoteFileChunk], + creationDate: Date?, + modificationDate: Date?, + account: Account, + options: NKRequestOptions, + currentNumChunksUpdateHandler: @escaping (Int) -> Void = { _ in }, + chunkCounter: @escaping (Int) -> Void = { _ in }, + chunkUploadStartHandler: @escaping ([RemoteFileChunk]) -> Void = { _ in }, + requestHandler: @escaping (UploadRequest) -> Void = { _ in }, + taskHandler: @escaping (URLSessionTask) -> Void = { _ in }, + progressHandler: @escaping (Progress) -> Void = { _ in }, + chunkUploadCompleteHandler: @escaping (RemoteFileChunk) -> Void = { _ in } + ) async -> ( + account: String, + fileChunks: [RemoteFileChunk]?, + file: NKFile?, + afError: AFError?, + remoteError: NKError + ) { + // Create temp directory for file and create chunks within it + let fm = FileManager.default + let tempDirectoryUrl = fm.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try! fm.createDirectory(atPath: tempDirectoryUrl.path, withIntermediateDirectories: true) + + // Access local file and gather metadata + let wholeLocalFile = localDirectoryPath + "/" + localFileName + let fileSize = try! fm.attributesOfItem(atPath: wholeLocalFile)[.size] as! Int + + let numChunks = Int(ceil(Double(fileSize) / Double(chunkSize))) + var remainingFileSize = fileSize + var chunks: [RemoteFileChunk] = [] + for chunkIndex in 0.. Date: Tue, 7 Jan 2025 18:44:24 +0900 Subject: [PATCH 07/86] Add logging when no valid root item on mock remote interface Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index bec38c73..541ee6e7 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -54,7 +54,10 @@ public class MockRemoteInterface: RemoteInterface { } func item(remotePath: String, account: Account) -> MockRemoteItem? { - guard let rootItem, !remotePath.isEmpty else { return nil } + guard let rootItem, !remotePath.isEmpty else { + print("Invalid root item or remote path, cannot get item in item tree.") + return nil + } let sanitisedPath = sanitisedPath(remotePath, account: account) guard sanitisedPath != "/" else { From 319e00470cb80f21bb0085f899c632f22c133627 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:44:38 +0900 Subject: [PATCH 08/86] Fix remaining file size in chunked upload implementation of MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 541ee6e7..a686055f 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -268,6 +268,7 @@ public class MockRemoteInterface: RemoteInterface { fileName: String(chunkIndex), size: Int64(min(chunkSize, remainingFileSize)) ) chunks.append(chunk) + remainingFileSize -= chunkSize } currentChunks[remoteChunkStoreFolderName] = chunks From 5ad26d534e520655bee26ed3b3c7f015583fdadb Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:44:47 +0900 Subject: [PATCH 09/86] Fix warning about unused response var Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index a686055f..6005f42f 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -272,7 +272,7 @@ public class MockRemoteInterface: RemoteInterface { } currentChunks[remoteChunkStoreFolderName] = chunks - let (_, ocId, etag, date, size, response, afError, remoteError) = await upload( + let (_, ocId, etag, date, size, _, afError, remoteError) = await upload( remotePath: remoteParentDirectoryPath + "/" + localFileName, localPath: wholeLocalFile, creationDate: creationDate, From 85a53b46a0c17e02f78bc532950d4517adb1b592 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:45:00 +0900 Subject: [PATCH 10/86] Add convenience method to get a root mock remote item Signed-off-by: Claudio Cambra f identifier Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteItem.swift | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Tests/Interface/MockRemoteItem.swift b/Tests/Interface/MockRemoteItem.swift index e708ac0b..e8051132 100644 --- a/Tests/Interface/MockRemoteItem.swift +++ b/Tests/Interface/MockRemoteItem.swift @@ -5,6 +5,7 @@ // Created by Claudio Cambra on 9/5/24. // +import FileProvider import Foundation import NextcloudFileProviderKit import NextcloudKit @@ -55,6 +56,20 @@ public class MockRemoteItem: Equatable { lhs.trashbinOriginalLocation == rhs.trashbinOriginalLocation } + public static func rootItem(account: Account) -> MockRemoteItem { + MockRemoteItem( + identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, + versionIdentifier: "root", + name: "root", + remotePath: account.davFilesUrl, + directory: true, + account: account.ncKitAccount, + username: account.username, + userId: account.id, + serverUrl: account.serverUrl + ) + } + public init( identifier: String, versionIdentifier: String = "0", From 8cc9751bf7d1515f168a5d05260e67d633aeea3a Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:48:07 +0900 Subject: [PATCH 11/86] Use convenience root item method in tests Signed-off-by: Claudio Cambra f use rootitem Signed-off-by: Claudio Cambra --- Tests/InterfaceTests/MockRemoteInterfaceTests.swift | 12 +----------- .../EnumeratorTests.swift | 11 +---------- .../ItemCreateTests.swift | 11 +---------- .../ItemDeleteTests.swift | 11 +---------- .../ItemFetchTests.swift | 11 +---------- .../ItemModifyTests.swift | 11 +---------- 6 files changed, 6 insertions(+), 61 deletions(-) diff --git a/Tests/InterfaceTests/MockRemoteInterfaceTests.swift b/Tests/InterfaceTests/MockRemoteInterfaceTests.swift index 29ce52d6..2b3edd29 100644 --- a/Tests/InterfaceTests/MockRemoteInterfaceTests.swift +++ b/Tests/InterfaceTests/MockRemoteInterfaceTests.swift @@ -13,17 +13,7 @@ final class MockRemoteInterfaceTests: XCTestCase { static let account = Account( user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: "root", - versionIdentifier: "root", - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) lazy var rootTrashItem = MockRemoteItem( identifier: "root", versionIdentifier: "root", diff --git a/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift b/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift index 6562cb24..34ea4331 100644 --- a/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift +++ b/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift @@ -34,16 +34,7 @@ final class EnumeratorTests: XCTestCase { super.setUp() Realm.Configuration.defaultConfiguration.inMemoryIdentifier = name - rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + rootItem = MockRemoteItem.rootItem(account: Self.account) remoteFolder = MockRemoteItem( identifier: "folder", diff --git a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift index f5534644..cc272647 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift @@ -18,16 +18,7 @@ final class ItemCreateTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) static let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) override func setUp() { diff --git a/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift b/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift index 0ffd786c..b21362ca 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift @@ -16,16 +16,7 @@ final class ItemDeleteTests: XCTestCase { static let account = Account( user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) lazy var rootTrashItem = MockRemoteItem( identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, name: "trash", diff --git a/Tests/NextcloudFileProviderKitTests/ItemFetchTests.swift b/Tests/NextcloudFileProviderKitTests/ItemFetchTests.swift index 2eae255c..1d1ea6bb 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemFetchTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemFetchTests.swift @@ -17,16 +17,7 @@ final class ItemFetchTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) static let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) override func setUp() { diff --git a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift index a55090d2..6134e832 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift @@ -18,16 +18,7 @@ final class ItemModifyTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "root", - remotePath: Self.account.davFilesUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) lazy var rootTrashItem = MockRemoteItem( identifier: NSFileProviderItemIdentifier.trashContainer.rawValue, name: "root", From 94cd66e9d5128bd5340f1d28d9740484cce66db4 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:51:13 +0900 Subject: [PATCH 12/86] Add convenience method to create root trash item Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteItem.swift | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Tests/Interface/MockRemoteItem.swift b/Tests/Interface/MockRemoteItem.swift index e8051132..b5c1f864 100644 --- a/Tests/Interface/MockRemoteItem.swift +++ b/Tests/Interface/MockRemoteItem.swift @@ -70,6 +70,20 @@ public class MockRemoteItem: Equatable { ) } + public static func rootTrashItem(account: Account) -> MockRemoteItem { + return MockRemoteItem( + identifier: NSFileProviderItemIdentifier.trashContainer.rawValue, + versionIdentifier: "root", + name: "root", + remotePath: account.trashUrl, + directory: true, + account: account.ncKitAccount, + username: account.username, + userId: account.id, + serverUrl: account.serverUrl + ) + } + public init( identifier: String, versionIdentifier: String = "0", From fac3db19f0595ed0d878e44990ca84d5043ac43e Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:51:53 +0900 Subject: [PATCH 13/86] Use convenience method to create root trash item in tests Signed-off-by: Claudio Cambra --- Tests/InterfaceTests/MockRemoteInterfaceTests.swift | 12 +----------- .../EnumeratorTests.swift | 11 +---------- .../ItemDeleteTests.swift | 11 +---------- .../ItemModifyTests.swift | 11 +---------- 4 files changed, 4 insertions(+), 41 deletions(-) diff --git a/Tests/InterfaceTests/MockRemoteInterfaceTests.swift b/Tests/InterfaceTests/MockRemoteInterfaceTests.swift index 2b3edd29..0a139ca9 100644 --- a/Tests/InterfaceTests/MockRemoteInterfaceTests.swift +++ b/Tests/InterfaceTests/MockRemoteInterfaceTests.swift @@ -14,17 +14,7 @@ final class MockRemoteInterfaceTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) - lazy var rootTrashItem = MockRemoteItem( - identifier: "root", - versionIdentifier: "root", - name: "root", - remotePath: Self.account.trashUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootTrashItem = MockRemoteItem.rootTrashItem(account: Self.account) override func tearDown() { rootItem.children = [] diff --git a/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift b/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift index 34ea4331..da32dca2 100644 --- a/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift +++ b/Tests/NextcloudFileProviderKitTests/EnumeratorTests.swift @@ -86,16 +86,7 @@ final class EnumeratorTests: XCTestCase { remoteItemB.parent = remoteFolder remoteItemC.parent = nil - rootTrashItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.trashContainer.rawValue, - name: "root", - remotePath: Self.account.trashUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + rootTrashItem = MockRemoteItem.rootTrashItem(account: Self.account) remoteTrashItemA = MockRemoteItem( identifier: "trashItemA", diff --git a/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift b/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift index b21362ca..f4cda9b7 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemDeleteTests.swift @@ -17,16 +17,7 @@ final class ItemDeleteTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) - lazy var rootTrashItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.rootContainer.rawValue, - name: "trash", - remotePath: Self.account.trashUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootTrashItem = MockRemoteItem.rootTrashItem(account: Self.account) static let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) override func setUp() { diff --git a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift index 6134e832..6f5dda5b 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift @@ -19,16 +19,7 @@ final class ItemModifyTests: XCTestCase { ) lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) - lazy var rootTrashItem = MockRemoteItem( - identifier: NSFileProviderItemIdentifier.trashContainer.rawValue, - name: "root", - remotePath: Self.account.trashUrl, - directory: true, - account: Self.account.ncKitAccount, - username: Self.account.username, - userId: Self.account.id, - serverUrl: Self.account.serverUrl - ) + lazy var rootTrashItem = MockRemoteItem.rootTrashItem(account: Self.account) var remoteFolder: MockRemoteItem! var remoteItem: MockRemoteItem! From 9c0f6cbb191cb926a1d13c7d9db1ac5d4d70d36a Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:52:45 +0900 Subject: [PATCH 14/86] Add UploadResult convenience struct Signed-off-by: Claudio Cambra --- .../Utilities/Upload.swift | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Sources/NextcloudFileProviderKit/Utilities/Upload.swift diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift new file mode 100644 index 00000000..b52854a9 --- /dev/null +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -0,0 +1,22 @@ +// +// Upload.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 2024-12-29. +// + +import Alamofire +import Foundation +import NextcloudKit + +struct UploadResult { + let ocId: String? + let chunks: [RemoteFileChunk]? + let etag: String? + let date: Date? + let size: Int64? + let afError: AFError? + let remoteError: NKError + var succeeded: Bool { remoteError == .success } +} + From fa7437cb4eb395407b9d7d2da9947e09c24e363c Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:53:07 +0900 Subject: [PATCH 15/86] Add starter convenience upload function to perform standard/chunked upload automatically Signed-off-by: Claudio Cambra --- .../Utilities/Upload.swift | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index b52854a9..e0675be5 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -8,6 +8,10 @@ import Alamofire import Foundation import NextcloudKit +import OSLog + +let defaultFileChunkSize = 10_000_000 // 10 MB +let uploadLogger = Logger(subsystem: Logger.subsystem, category: "upload") struct UploadResult { let ocId: String? @@ -20,3 +24,93 @@ struct UploadResult { var succeeded: Bool { remoteError == .success } } +func upload( + fileLocatedAt localFileUrl: URL, + toRemotePath remotePath: String, + usingRemoteInterface remoteInterface: RemoteInterface, + withAccount account: Account, + inChunksSized chunkSize: Int = defaultFileChunkSize, + creationDate: Date? = nil, + modificationDate: Date? = nil, + options: NKRequestOptions = .init(), + requestHandler: @escaping (UploadRequest) -> Void = { _ in }, + taskHandler: @escaping (URLSessionTask) -> Void = { _ in }, + progressHandler: @escaping (Progress) -> Void = { _ in } +) async -> UploadResult { + let localPath = localFileUrl.path + let fileSize = + (try? FileManager.default.attributesOfItem(atPath: localPath)[.size] as? Int64) ?? 0 + guard fileSize > chunkSize else { + let (_, ocId, etag, date, size, _, afError, remoteError) = await remoteInterface.upload( + remotePath: remotePath, + localPath: localFileUrl.path, + creationDate: creationDate, + modificationDate: modificationDate, + account: account, + options: options, + requestHandler: requestHandler, + taskHandler: taskHandler, + progressHandler: progressHandler + ) + + return UploadResult( + ocId: ocId, + chunks: nil, + etag: etag, + date: date as? Date, + size: size, + afError: afError, + remoteError: remoteError + ) + } + + let localFileName = localFileUrl.lastPathComponent + let localParentDirectoryPath = localFileUrl.deletingLastPathComponent().path + let remoteParentDirectoryPath = (remotePath as NSString).deletingLastPathComponent as String + // TODO: Pick up where left off for past failed chunked uploads + let (_, chunks, file, afError, remoteError) = await remoteInterface.chunkedUpload( + localDirectoryPath: localParentDirectoryPath, + localFileName: localFileName, + remoteParentDirectoryPath: remoteParentDirectoryPath, + remoteChunkStoreFolderName: UUID().uuidString, + chunkSize: defaultFileChunkSize, + remainingChunks: [], + creationDate: creationDate, + modificationDate: modificationDate, + account: account, + options: options, + currentNumChunksUpdateHandler: { _ in }, + chunkCounter: { currentChunk in + uploadLogger.info( + """ + \(localFileName, privacy: .public) current chunk: \(currentChunk, privacy: .public) + """ + ) + }, + chunkUploadStartHandler: { _ in + uploadLogger.info("\(localFileName, privacy: .public) uploading chunk") + }, + requestHandler: requestHandler, + taskHandler: taskHandler, + progressHandler: progressHandler, + chunkUploadCompleteHandler: { uploadedChunk in + uploadLogger.info( + """ + \(localFileName, privacy: .public) uploaded chunk: + \(uploadedChunk.fileName, privacy: .public) + (\(uploadedChunk.size, privacy: .public)) + """ + ) + } + ) + + return UploadResult( + ocId: file?.name, + chunks: chunks, + etag: file?.etag, + date: file?.date, + size: file?.size, + afError: afError, + remoteError: remoteError + ) +} From 6cb61f14c0bee7058c8eb3e678e9d792c55f1885 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 18:53:22 +0900 Subject: [PATCH 16/86] Add basic test to check standard upload in upload convenience func Signed-off-by: Claudio Cambra --- .../UploadTests.swift | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index 82d752ce..1bea5498 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -5,6 +5,7 @@ // Created by Claudio Cambra on 2025-01-07. // +import TestInterface import XCTest @testable import NextcloudFileProviderKit @@ -22,4 +23,28 @@ final class UploadTests: XCTestCase { ) XCTAssertTrue(uploadResult.succeeded) } + + func testStandardUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let account = Account(user: "user", id: "id", serverUrl: "test.cloud.com", password: "1234") + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) + let remotePath = account.davFilesUrl + "/file.txt" + let result = await NextcloudFileProviderKit.upload( + fileLocatedAt: fileUrl, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account + ) + + XCTAssertTrue(result.succeeded) + XCTAssertNil(result.chunks) + XCTAssertEqual(result.size, Int64(data.count)) + XCTAssertNotNil(result.ocId) + XCTAssertNotNil(result.etag) + } } From a59bd09baca074de3347b7a9e4fa3c6796081576 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 19:13:13 +0900 Subject: [PATCH 17/86] Use correct chunkSize value in uplaod convenience func Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Utilities/Upload.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index e0675be5..d883a9cd 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -73,7 +73,7 @@ func upload( localFileName: localFileName, remoteParentDirectoryPath: remoteParentDirectoryPath, remoteChunkStoreFolderName: UUID().uuidString, - chunkSize: defaultFileChunkSize, + chunkSize: chunkSize, remainingChunks: [], creationDate: creationDate, modificationDate: modificationDate, From 9dc801f0ee3ded36a8b5cb01325b964926619966 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 19:13:42 +0900 Subject: [PATCH 18/86] Add simple test for upload convenience func chunked upload Signed-off-by: Claudio Cambra --- .../UploadTests.swift | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index 1bea5498..5ceaa06d 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -47,4 +47,30 @@ final class UploadTests: XCTestCase { XCTAssertNotNil(result.ocId) XCTAssertNotNil(result.etag) } + + func testChunkedUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let account = Account(user: "user", id: "id", serverUrl: "test.cloud.com", password: "1234") + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) + let remotePath = account.davFilesUrl + "/file.txt" + let result = await NextcloudFileProviderKit.upload( + fileLocatedAt: fileUrl, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + inChunksSized: 3 + ) + + XCTAssertTrue(result.succeeded) + XCTAssertNotNil(result.chunks) + XCTAssertEqual(result.chunks?.count, 3) + XCTAssertEqual(result.size, Int64(data.count)) + XCTAssertNotNil(result.ocId) + XCTAssertNotNil(result.etag) + } } From 05e446b38e3d91b87a5ee3edd8460e350ec0625a Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Tue, 7 Jan 2025 19:25:21 +0900 Subject: [PATCH 19/86] Start chunk name from 1 imitating the real server Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 6005f42f..8cfdf93b 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -265,7 +265,7 @@ public class MockRemoteInterface: RemoteInterface { var chunks: [RemoteFileChunk] = [] for chunkIndex in 0.. Date: Tue, 7 Jan 2025 19:26:08 +0900 Subject: [PATCH 20/86] Make chunk upload id an argument in convenience upload func Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Utilities/Upload.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index d883a9cd..3c75a054 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -30,6 +30,7 @@ func upload( usingRemoteInterface remoteInterface: RemoteInterface, withAccount account: Account, inChunksSized chunkSize: Int = defaultFileChunkSize, + usingChunkUploadId chunkUploadId: String = UUID().uuidString, creationDate: Date? = nil, modificationDate: Date? = nil, options: NKRequestOptions = .init(), @@ -72,7 +73,7 @@ func upload( localDirectoryPath: localParentDirectoryPath, localFileName: localFileName, remoteParentDirectoryPath: remoteParentDirectoryPath, - remoteChunkStoreFolderName: UUID().uuidString, + remoteChunkStoreFolderName: chunkUploadId, chunkSize: chunkSize, remainingChunks: [], creationDate: creationDate, From e7fa1e06925428718735ff65a37c5fb36d8e017b Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 00:04:12 +0900 Subject: [PATCH 21/86] Set chunks at initialisation time in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 8cfdf93b..ac33a285 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -260,15 +260,15 @@ public class MockRemoteInterface: RemoteInterface { let wholeLocalFile = localDirectoryPath + "/" + localFileName let fileSize = try! fm.attributesOfItem(atPath: wholeLocalFile)[.size] as! Int - let numChunks = Int(ceil(Double(fileSize) / Double(chunkSize))) var remainingFileSize = fileSize - var chunks: [RemoteFileChunk] = [] - for chunkIndex in 0.. Date: Wed, 8 Jan 2025 00:04:25 +0900 Subject: [PATCH 22/86] Call chunkUploadStartedHandler in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index ac33a285..6298b163 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -271,6 +271,7 @@ public class MockRemoteInterface: RemoteInterface { return chunk } currentChunks[remoteChunkStoreFolderName] = chunks + chunkUploadStartHandler(chunks) let (_, ocId, etag, date, size, _, afError, remoteError) = await upload( remotePath: remoteParentDirectoryPath + "/" + localFileName, From fb4f3d944babb08b382d0f53a2487284b0ced7b7 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 00:04:40 +0900 Subject: [PATCH 23/86] Pass all relevant arguments to inner upload during chunk upload in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 6298b163..8bb3da62 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -278,8 +278,11 @@ public class MockRemoteInterface: RemoteInterface { localPath: wholeLocalFile, creationDate: creationDate, modificationDate: modificationDate, - - account: account + account: account, + options: options, + requestHandler: requestHandler, + taskHandler: taskHandler, + progressHandler: progressHandler ) let file = NKFile() file.fileName = localFileName From b5b35ae8fe5f39d56302d3e4d140ca0c4d8edbd7 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 00:04:49 +0900 Subject: [PATCH 24/86] Call chunkUploadCompleteHandler in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 8bb3da62..b1b123cb 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -284,6 +284,8 @@ public class MockRemoteInterface: RemoteInterface { taskHandler: taskHandler, progressHandler: progressHandler ) + chunks.forEach { chunkUploadCompleteHandler($0) } + let file = NKFile() file.fileName = localFileName file.etag = etag ?? "" From 1bf5c2bd79a0add051dca3412c8099f5ed99b1e9 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 00:06:26 +0900 Subject: [PATCH 25/86] Allow passing in chunkUploadCompleteHandler Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Utilities/Upload.swift | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 3c75a054..f8952fa1 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -36,7 +36,8 @@ func upload( options: NKRequestOptions = .init(), requestHandler: @escaping (UploadRequest) -> Void = { _ in }, taskHandler: @escaping (URLSessionTask) -> Void = { _ in }, - progressHandler: @escaping (Progress) -> Void = { _ in } + progressHandler: @escaping (Progress) -> Void = { _ in }, + chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void = { _ in } ) async -> UploadResult { let localPath = localFileUrl.path let fileSize = @@ -94,15 +95,7 @@ func upload( requestHandler: requestHandler, taskHandler: taskHandler, progressHandler: progressHandler, - chunkUploadCompleteHandler: { uploadedChunk in - uploadLogger.info( - """ - \(localFileName, privacy: .public) uploaded chunk: - \(uploadedChunk.fileName, privacy: .public) - (\(uploadedChunk.size, privacy: .public)) - """ - ) - } + chunkUploadCompleteHandler: chunkUploadCompleteHandler ) return UploadResult( From e9feb47b1862ab3b6f7ca8fe4d475861d1bff98e Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 00:13:05 +0900 Subject: [PATCH 26/86] Only call chunkUploadCompleteHandler once in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index b1b123cb..1aedc1d6 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -266,7 +266,6 @@ public class MockRemoteInterface: RemoteInterface { let chunk = RemoteFileChunk( fileName: String(chunkIndex + 1), size: Int64(min(chunkSize, remainingFileSize)) ) - chunkUploadCompleteHandler(chunk) remainingFileSize -= chunkSize return chunk } From 9ae04df31e812dea2e29d8622591e4b45998af26 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:25:36 +0900 Subject: [PATCH 27/86] Add RemoteFileChunk database object type Signed-off-by: Claudio Cambra --- .../Metadata/RemoteFileChunk.swift | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift diff --git a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift new file mode 100644 index 00000000..e14e156f --- /dev/null +++ b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift @@ -0,0 +1,17 @@ +// +// RemoteFileChunk.swift +// NextcloudFileProviderKit +// +// Created by Claudio Cambra on 2025-01-08. +// + +import Foundation +import NextcloudKit +import RealmSwift + +public class RemoteFileChunk: Object { + @Persisted public var fileName: String + @Persisted public var size: Int64 + @Persisted public var uploadUuid: String +} + From 04e42f3bb069d3ac2cc70ff45ae8af3d5e69956b Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:26:30 +0900 Subject: [PATCH 28/86] Remove previous RemoteFileChunk typealias Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Interface/RemoteInterface.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift index 6811d5c8..81e0719c 100644 --- a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift @@ -20,8 +20,6 @@ public enum AuthenticationAttemptResultState: Int { case authenticationError, connectionError, success } -public typealias RemoteFileChunk = (fileName: String, size: Int64) - public protocol RemoteInterface { func setDelegate(_ delegate: NextcloudKitDelegate) From 49275d0e880cecf1eb25e8dc4a0bdc49866b1dca Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:26:55 +0900 Subject: [PATCH 29/86] Add convenience to get array of remotefilechunk from the nckit file chunk tuples Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift index e14e156f..8eb03bdb 100644 --- a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift +++ b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift @@ -13,5 +13,9 @@ public class RemoteFileChunk: Object { @Persisted public var fileName: String @Persisted public var size: Int64 @Persisted public var uploadUuid: String + + static func fromNcKitChunks(_ chunks: [(fileName: String, size: Int64)]) -> [RemoteFileChunk] { + chunks.map { RemoteFileChunk(ncKitChunk: $0) } + } } From 8b31c17106b9d192f9b2885a633bcae700e68ad7 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:27:08 +0900 Subject: [PATCH 30/86] Add convenience init for remotefilechunk based on nckit tuple Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift index 8eb03bdb..21790e61 100644 --- a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift +++ b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift @@ -17,5 +17,11 @@ public class RemoteFileChunk: Object { static func fromNcKitChunks(_ chunks: [(fileName: String, size: Int64)]) -> [RemoteFileChunk] { chunks.map { RemoteFileChunk(ncKitChunk: $0) } } + + convenience init(ncKitChunk: (fileName: String, size: Int64)) { + self.init() + fileName = ncKitChunk.fileName + size = ncKitChunk.size + } } From dec57d1178869500313ca3645fbc7db65aa71289 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:27:20 +0900 Subject: [PATCH 31/86] Add convenience converted from remotefilechunk to nckit tuple Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift index 21790e61..a649e832 100644 --- a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift +++ b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift @@ -23,5 +23,9 @@ public class RemoteFileChunk: Object { fileName = ncKitChunk.fileName size = ncKitChunk.size } + + func toNcKitChunk() -> (fileName: String, size: Int64) { + (fileName, size) + } } From 0f97512b49410071ee91c4cd5aea1b661104cdc5 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:27:38 +0900 Subject: [PATCH 32/86] Add convenience converter from array of nckit chunk tuples to remote file chunks Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift index a649e832..4d57d7f5 100644 --- a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift +++ b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift @@ -29,3 +29,8 @@ public class RemoteFileChunk: Object { } } +extension Array { + func toNcKitChunks() -> [(fileName: String, size: Int64)] { + map { ($0.fileName, $0.size) } + } +} From 029cf9b34fa0b080042197b43cb4121b96accd8e Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:27:56 +0900 Subject: [PATCH 33/86] Add method to convert array of managed remotefilechunks to unmanaged Signed-off-by: Claudio Cambra --- .../Extensions/Results+Extensions.swift | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift b/Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift index 97a9e041..990a9e5f 100644 --- a/Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift +++ b/Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift @@ -8,8 +8,17 @@ import Realm import RealmSwift +extension Results where Element: RemoteFileChunk { + func toUnmanagedResults() -> [RemoteFileChunk] { + return map { RemoteFileChunk(value: $0) } + } +} + extension Results where Element: ItemMetadata { func toUnmanagedResults() -> [ItemMetadata] { return map { ItemMetadata(value: $0) } } } + + + From fbc801fff3f8060bebd30c7f9aaf7b128e42b981 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:28:05 +0900 Subject: [PATCH 34/86] Register RemoteFileChunk in database init Signed-off-by: Claudio Cambra --- .../Database/FilesDatabaseManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift index 87aec5bd..603ab211 100644 --- a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift +++ b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift @@ -89,7 +89,7 @@ public class FilesDatabaseManager { } }, - objectTypes: [ItemMetadata.self] + objectTypes: [ItemMetadata.self, RemoteFileChunk.self] ) self.init(realmConfig: config) } From eb3aae023056e5d272e125901366eca57b554d8f Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:28:19 +0900 Subject: [PATCH 35/86] Fix uses of remotefilechunk in nckit remoteinterface Signed-off-by: Claudio Cambra --- .../Interface/NextcloudKit+RemoteInterface.swift | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 3cde6b48..72b5183c 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -113,13 +113,13 @@ extension NextcloudKit: RemoteInterface { creationDate: creationDate, serverUrl: remoteParentDirectoryPath, chunkFolder: remoteChunkStoreFolderName, - filesChunk: remainingChunks, + filesChunk: remainingChunks.toNcKitChunks(), chunkSize: chunkSize, account: account.ncKitAccount, options: options, numChunks: currentNumChunksUpdateHandler, counterChunk: chunkCounter, - start: chunkUploadStartHandler, + start: { chunkUploadStartHandler(RemoteFileChunk.fromNcKitChunks($0)) }, requestHandler: requestHandler, taskHandler: taskHandler, progressHandler: { totalBytesExpected, totalBytes, fractionCompleted in @@ -127,9 +127,9 @@ extension NextcloudKit: RemoteInterface { currentProgress.completedUnitCount = totalBytes progressHandler(currentProgress) }, - uploaded: chunkUploadCompleteHandler - ) { account, filesChunk, file, afError, error in - let chunks = filesChunk as [RemoteFileChunk]? + uploaded: { chunkUploadCompleteHandler(RemoteFileChunk(ncKitChunk: $0)) } + ) { account, receivedChunks, file, afError, error in + let chunks = RemoteFileChunk.fromNcKitChunks(receivedChunks ?? []) continuation.resume(returning: (account, chunks, file, afError, error)) } } From 45ba02f44df795121b52f589f71f8ccbb3e9fe3c Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:37:24 +0900 Subject: [PATCH 36/86] Require remoteChunkStoreFolderName for remote file chunk Signed-off-by: Claudio Cambra --- .../Metadata/RemoteFileChunk.swift | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift index 4d57d7f5..029897b8 100644 --- a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift +++ b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift @@ -12,16 +12,29 @@ import RealmSwift public class RemoteFileChunk: Object { @Persisted public var fileName: String @Persisted public var size: Int64 - @Persisted public var uploadUuid: String + @Persisted public var remoteChunkStoreFolderName: String - static func fromNcKitChunks(_ chunks: [(fileName: String, size: Int64)]) -> [RemoteFileChunk] { - chunks.map { RemoteFileChunk(ncKitChunk: $0) } + static func fromNcKitChunks( + _ chunks: [(fileName: String, size: Int64)], remoteChunkStoreFolderName: String + ) -> [RemoteFileChunk] { + chunks.map { + RemoteFileChunk(ncKitChunk: $0, remoteChunkStoreFolderName: remoteChunkStoreFolderName) + } } - convenience init(ncKitChunk: (fileName: String, size: Int64)) { + convenience init(ncKitChunk: (fileName: String, size: Int64), remoteChunkStoreFolderName: String) { + self.init( + fileName: ncKitChunk.fileName, + size: ncKitChunk.size, + remoteChunkStoreFolderName: remoteChunkStoreFolderName + ) + } + + convenience init(fileName: String, size: Int64, remoteChunkStoreFolderName: String) { self.init() - fileName = ncKitChunk.fileName - size = ncKitChunk.size + self.fileName = fileName + self.size = size + self.remoteChunkStoreFolderName = remoteChunkStoreFolderName } func toNcKitChunk() -> (fileName: String, size: Int64) { From a21b597cae0544aed88a32ba6dfa534b97985237 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:37:59 +0900 Subject: [PATCH 37/86] Fix handling of remote file chunks in nckit remote interface, again Signed-off-by: Claudio Cambra --- .../NextcloudKit+RemoteInterface.swift | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 72b5183c..1b5cd765 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -119,7 +119,12 @@ extension NextcloudKit: RemoteInterface { options: options, numChunks: currentNumChunksUpdateHandler, counterChunk: chunkCounter, - start: { chunkUploadStartHandler(RemoteFileChunk.fromNcKitChunks($0)) }, + start: { processedChunks in + let chunks = RemoteFileChunk.fromNcKitChunks( + processedChunks, remoteChunkStoreFolderName: remoteChunkStoreFolderName + ) + chunkUploadStartHandler(chunks) + }, requestHandler: requestHandler, taskHandler: taskHandler, progressHandler: { totalBytesExpected, totalBytes, fractionCompleted in @@ -127,9 +132,17 @@ extension NextcloudKit: RemoteInterface { currentProgress.completedUnitCount = totalBytes progressHandler(currentProgress) }, - uploaded: { chunkUploadCompleteHandler(RemoteFileChunk(ncKitChunk: $0)) } + uploaded: { uploadedChunk in + let chunk = RemoteFileChunk( + ncKitChunk: uploadedChunk, + remoteChunkStoreFolderName: remoteChunkStoreFolderName + ) + chunkUploadCompleteHandler(chunk) + } ) { account, receivedChunks, file, afError, error in - let chunks = RemoteFileChunk.fromNcKitChunks(receivedChunks ?? []) + let chunks = RemoteFileChunk.fromNcKitChunks( + receivedChunks ?? [], remoteChunkStoreFolderName: remoteChunkStoreFolderName + ) continuation.resume(returning: (account, chunks, file, afError, error)) } } From 195d356f96bff797d940b6242777c1b0579d783e Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 01:41:22 +0900 Subject: [PATCH 38/86] Keep track of uploaded and yet to upload chunks in chunked upload Signed-off-by: Claudio Cambra --- .../Utilities/Upload.swift | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index f8952fa1..8d5f1593 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -9,6 +9,7 @@ import Alamofire import Foundation import NextcloudKit import OSLog +import RealmSwift let defaultFileChunkSize = 10_000_000 // 10 MB let uploadLogger = Logger(subsystem: Logger.subsystem, category: "upload") @@ -31,6 +32,7 @@ func upload( withAccount account: Account, inChunksSized chunkSize: Int = defaultFileChunkSize, usingChunkUploadId chunkUploadId: String = UUID().uuidString, + dbManager: FilesDatabaseManager = .shared, creationDate: Date? = nil, modificationDate: Date? = nil, options: NKRequestOptions = .init(), @@ -69,14 +71,20 @@ func upload( let localFileName = localFileUrl.lastPathComponent let localParentDirectoryPath = localFileUrl.deletingLastPathComponent().path let remoteParentDirectoryPath = (remotePath as NSString).deletingLastPathComponent as String - // TODO: Pick up where left off for past failed chunked uploads + + let remainingChunks = dbManager + .ncDatabase() + .objects(RemoteFileChunk.self) + .toUnmanagedResults() + .filter { $0.uploadUuid == chunkUploadId } + let (_, chunks, file, afError, remoteError) = await remoteInterface.chunkedUpload( localDirectoryPath: localParentDirectoryPath, localFileName: localFileName, remoteParentDirectoryPath: remoteParentDirectoryPath, remoteChunkStoreFolderName: chunkUploadId, chunkSize: chunkSize, - remainingChunks: [], + remainingChunks: remainingChunks, creationDate: creationDate, modificationDate: modificationDate, account: account, @@ -89,13 +97,41 @@ func upload( """ ) }, - chunkUploadStartHandler: { _ in + chunkUploadStartHandler: { chunks in uploadLogger.info("\(localFileName, privacy: .public) uploading chunk") + let db = dbManager.ncDatabase() + do { + try db.write { db.add(chunks) } + } catch let error { + uploadLogger.error( + """ + Could not write chunks to db, won't be able to resume upload if transfer stops. + \(error.localizedDescription, privacy: .public) + """ + ) + } }, requestHandler: requestHandler, taskHandler: taskHandler, progressHandler: progressHandler, - chunkUploadCompleteHandler: chunkUploadCompleteHandler + chunkUploadCompleteHandler: { chunk in + let db = dbManager.ncDatabase() + do { + try db.write { + let dbChunks = db.objects(RemoteFileChunk.self) + dbChunks + .filter { $0.uploadUuid == chunkUploadId && $0.fileName == chunk.fileName } + .forEach { db.delete($0) } } + } catch let error { + uploadLogger.error( + """ + Could not delete chunks in db, won't resume upload correctly if transfer stops. + \(error.localizedDescription, privacy: .public) + """ + ) + } + chunkUploadCompleteHandler(chunk) + } ) return UploadResult( From f5ecad37c7852130bae49c15530beb353b7db47b Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 02:13:51 +0900 Subject: [PATCH 39/86] Make methods in RemoteFileChunk public Signed-off-by: Claudio Cambra --- .../Metadata/RemoteFileChunk.swift | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift index 029897b8..c04d0b10 100644 --- a/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift +++ b/Sources/NextcloudFileProviderKit/Metadata/RemoteFileChunk.swift @@ -14,7 +14,7 @@ public class RemoteFileChunk: Object { @Persisted public var size: Int64 @Persisted public var remoteChunkStoreFolderName: String - static func fromNcKitChunks( + public static func fromNcKitChunks( _ chunks: [(fileName: String, size: Int64)], remoteChunkStoreFolderName: String ) -> [RemoteFileChunk] { chunks.map { @@ -22,7 +22,9 @@ public class RemoteFileChunk: Object { } } - convenience init(ncKitChunk: (fileName: String, size: Int64), remoteChunkStoreFolderName: String) { + public convenience init( + ncKitChunk: (fileName: String, size: Int64), remoteChunkStoreFolderName: String + ) { self.init( fileName: ncKitChunk.fileName, size: ncKitChunk.size, @@ -30,7 +32,7 @@ public class RemoteFileChunk: Object { ) } - convenience init(fileName: String, size: Int64, remoteChunkStoreFolderName: String) { + public convenience init(fileName: String, size: Int64, remoteChunkStoreFolderName: String) { self.init() self.fileName = fileName self.size = size From af8bf5ba115ff3325387020fc8aa266e85c71de3 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 02:14:19 +0900 Subject: [PATCH 40/86] Rename uses of uploadUuid to new remoteChunkFolder property Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Utilities/Upload.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 8d5f1593..49677e36 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -76,7 +76,7 @@ func upload( .ncDatabase() .objects(RemoteFileChunk.self) .toUnmanagedResults() - .filter { $0.uploadUuid == chunkUploadId } + .filter { $0.remoteChunkStoreFolderName == chunkUploadId } let (_, chunks, file, afError, remoteError) = await remoteInterface.chunkedUpload( localDirectoryPath: localParentDirectoryPath, @@ -120,7 +120,10 @@ func upload( try db.write { let dbChunks = db.objects(RemoteFileChunk.self) dbChunks - .filter { $0.uploadUuid == chunkUploadId && $0.fileName == chunk.fileName } + .filter { + $0.remoteChunkStoreFolderName == chunkUploadId && + $0.fileName == chunk.fileName + } .forEach { db.delete($0) } } } catch let error { uploadLogger.error( From fc82890cae7bb6393227422fc8f2b4c71e1a617e Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 02:14:35 +0900 Subject: [PATCH 41/86] Fix and clean up chunk creation in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 1aedc1d6..75b990dc 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -263,11 +263,12 @@ public class MockRemoteInterface: RemoteInterface { var remainingFileSize = fileSize let numChunks = Int(ceil(Double(fileSize) / Double(chunkSize))) let chunks = (0.. Date: Wed, 8 Jan 2025 02:14:53 +0900 Subject: [PATCH 42/86] Ensure chunks stay unmanaged in upload procedure Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Utilities/Upload.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 49677e36..ef1624de 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -101,7 +101,7 @@ func upload( uploadLogger.info("\(localFileName, privacy: .public) uploading chunk") let db = dbManager.ncDatabase() do { - try db.write { db.add(chunks) } + try db.write { db.add(chunks.map { RemoteFileChunk(value: $0) }) } } catch let error { uploadLogger.error( """ From d65da185ab7069645a91ec08d87f8b9bd612afcc Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 02:15:15 +0900 Subject: [PATCH 43/86] Pass test database in upload tests into upload func Signed-off-by: Claudio Cambra --- .../UploadTests.swift | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index 5ceaa06d..8d5c16ff 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -5,11 +5,19 @@ // Created by Claudio Cambra on 2025-01-07. // +import RealmSwift import TestInterface import XCTest @testable import NextcloudFileProviderKit final class UploadTests: XCTestCase { + let account = Account(user: "user", id: "id", serverUrl: "test.cloud.com", password: "1234") + let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) + + override func setUp() { + super.setUp() + Realm.Configuration.defaultConfiguration.inMemoryIdentifier = name + } func testSucceededUploadResult() { let uploadResult = UploadResult( @@ -30,7 +38,6 @@ final class UploadTests: XCTestCase { let data = Data(repeating: 1, count: 8) try data.write(to: fileUrl) - let account = Account(user: "user", id: "id", serverUrl: "test.cloud.com", password: "1234") let remoteInterface = MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) let remotePath = account.davFilesUrl + "/file.txt" @@ -38,7 +45,8 @@ final class UploadTests: XCTestCase { fileLocatedAt: fileUrl, toRemotePath: remotePath, usingRemoteInterface: remoteInterface, - withAccount: account + withAccount: account, + dbManager: dbManager ) XCTAssertTrue(result.succeeded) @@ -54,7 +62,6 @@ final class UploadTests: XCTestCase { let data = Data(repeating: 1, count: 8) try data.write(to: fileUrl) - let account = Account(user: "user", id: "id", serverUrl: "test.cloud.com", password: "1234") let remoteInterface = MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) let remotePath = account.davFilesUrl + "/file.txt" @@ -63,7 +70,8 @@ final class UploadTests: XCTestCase { toRemotePath: remotePath, usingRemoteInterface: remoteInterface, withAccount: account, - inChunksSized: 3 + inChunksSized: 3, + dbManager: dbManager ) XCTAssertTrue(result.succeeded) From 64251439250cd4a4b6835fdbd32cc27549ca115a Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 10:22:33 +0800 Subject: [PATCH 44/86] Remove UploadResult struct as it is not necessary Signed-off-by: Claudio Cambra --- .../Utilities/Upload.swift | 25 ++++++++----------- .../UploadTests.swift | 17 ++----------- 2 files changed, 13 insertions(+), 29 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index ef1624de..2e1cdc01 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -14,17 +14,6 @@ import RealmSwift let defaultFileChunkSize = 10_000_000 // 10 MB let uploadLogger = Logger(subsystem: Logger.subsystem, category: "upload") -struct UploadResult { - let ocId: String? - let chunks: [RemoteFileChunk]? - let etag: String? - let date: Date? - let size: Int64? - let afError: AFError? - let remoteError: NKError - var succeeded: Bool { remoteError == .success } -} - func upload( fileLocatedAt localFileUrl: URL, toRemotePath remotePath: String, @@ -40,7 +29,15 @@ func upload( taskHandler: @escaping (URLSessionTask) -> Void = { _ in }, progressHandler: @escaping (Progress) -> Void = { _ in }, chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void = { _ in } -) async -> UploadResult { +) async -> ( + ocId: String?, + chunks: [RemoteFileChunk]?, + etag: String?, + date: Date?, + size: Int64?, + afError: AFError?, + remoteError: NKError +) { let localPath = localFileUrl.path let fileSize = (try? FileManager.default.attributesOfItem(atPath: localPath)[.size] as? Int64) ?? 0 @@ -57,7 +54,7 @@ func upload( progressHandler: progressHandler ) - return UploadResult( + return ( ocId: ocId, chunks: nil, etag: etag, @@ -137,7 +134,7 @@ func upload( } ) - return UploadResult( + return ( ocId: file?.name, chunks: chunks, etag: file?.etag, diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index 8d5c16ff..ac6ae57f 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -19,19 +19,6 @@ final class UploadTests: XCTestCase { Realm.Configuration.defaultConfiguration.inMemoryIdentifier = name } - func testSucceededUploadResult() { - let uploadResult = UploadResult( - ocId: nil, - chunks: nil, - etag: nil, - date: nil, - size: nil, - afError: nil, - remoteError: .success - ) - XCTAssertTrue(uploadResult.succeeded) - } - func testStandardUpload() async throws { let fileUrl = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) @@ -49,7 +36,7 @@ final class UploadTests: XCTestCase { dbManager: dbManager ) - XCTAssertTrue(result.succeeded) + XCTAssertEqual(result.remoteError, .success) XCTAssertNil(result.chunks) XCTAssertEqual(result.size, Int64(data.count)) XCTAssertNotNil(result.ocId) @@ -74,7 +61,7 @@ final class UploadTests: XCTestCase { dbManager: dbManager ) - XCTAssertTrue(result.succeeded) + XCTAssertEqual(result.remoteError, .success) XCTAssertNotNil(result.chunks) XCTAssertEqual(result.chunks?.count, 3) XCTAssertEqual(result.size, Int64(data.count)) From b54d331b61ceb189dc9e6261af7f6f125fe43966 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 10:54:03 +0800 Subject: [PATCH 45/86] Fix remainingFileSize calculation in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 75b990dc..efcbfdfd 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -263,7 +263,7 @@ public class MockRemoteInterface: RemoteInterface { var remainingFileSize = fileSize let numChunks = Int(ceil(Double(fileSize) / Double(chunkSize))) let chunks = (0.. Date: Wed, 8 Jan 2025 10:54:57 +0800 Subject: [PATCH 46/86] Simulate resumed chunk upload in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index efcbfdfd..33405e5d 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -270,6 +270,7 @@ public class MockRemoteInterface: RemoteInterface { remoteChunkStoreFolderName: remoteChunkStoreFolderName ) } + let preexistingChunks = currentChunks[remoteChunkStoreFolderName] ?? [] currentChunks[remoteChunkStoreFolderName] = chunks chunkUploadStartHandler(chunks) @@ -284,7 +285,11 @@ public class MockRemoteInterface: RemoteInterface { taskHandler: taskHandler, progressHandler: progressHandler ) - chunks.forEach { chunkUploadCompleteHandler($0) } + chunks.forEach { chunk in + if !preexistingChunks.contains(where: { $0.fileName == chunk.fileName }) { + chunkUploadCompleteHandler(chunk) + } + } let file = NKFile() file.fileName = localFileName From 730747883493a1b855b92d97d72b08d0df9b6692 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 10:56:15 +0800 Subject: [PATCH 47/86] Enhance testing of chunked uploads by testing for correct chunk handling Signed-off-by: Claudio Cambra --- .../UploadTests.swift | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index ac6ae57f..23dfdc51 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -52,20 +52,37 @@ final class UploadTests: XCTestCase { let remoteInterface = MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) let remotePath = account.davFilesUrl + "/file.txt" + let chunkSize = 3 + var uploadedChunks = [RemoteFileChunk]() let result = await NextcloudFileProviderKit.upload( fileLocatedAt: fileUrl, toRemotePath: remotePath, usingRemoteInterface: remoteInterface, withAccount: account, - inChunksSized: 3, - dbManager: dbManager + inChunksSized: chunkSize, + dbManager: dbManager, + chunkUploadCompleteHandler: { uploadedChunks.append($0) } ) + let resultChunks = try XCTUnwrap(result.chunks) + let expectedChunkCount = Int(ceil(Double(data.count) / Double(chunkSize))) XCTAssertEqual(result.remoteError, .success) - XCTAssertNotNil(result.chunks) - XCTAssertEqual(result.chunks?.count, 3) + XCTAssertEqual(resultChunks.count, expectedChunkCount) XCTAssertEqual(result.size, Int64(data.count)) XCTAssertNotNil(result.ocId) XCTAssertNotNil(result.etag) + + XCTAssertEqual(uploadedChunks.count, resultChunks.count) + + let firstUploadedChunk = try XCTUnwrap(uploadedChunks.first) + let firstUploadedChunkNameInt = try XCTUnwrap(Int(firstUploadedChunk.fileName)) + let lastUploadedChunk = try XCTUnwrap(uploadedChunks.last) + let lastUploadedChunkNameInt = try XCTUnwrap(Int(lastUploadedChunk.fileName)) + XCTAssertEqual(firstUploadedChunkNameInt, 1) + XCTAssertEqual(lastUploadedChunkNameInt, expectedChunkCount) + XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) + XCTAssertEqual( + Int(lastUploadedChunk.size), data.count - (lastUploadedChunkNameInt * chunkSize) + ) } } From 8dff7b20ccfc772a4bf776ac400b83dc54b3a0f7 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 10:56:30 +0800 Subject: [PATCH 48/86] Add test for resuming interrupted chunked uploads Signed-off-by: Claudio Cambra --- .../UploadTests.swift | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index 23dfdc51..9383b0fe 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -85,4 +85,59 @@ final class UploadTests: XCTestCase { Int(lastUploadedChunk.size), data.count - (lastUploadedChunkNameInt * chunkSize) ) } + + func testResumingInterruptedChunkedUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) + let chunkSize = 3 + let uploadUuid = UUID().uuidString + let previousUploadedChunkNum = 1 + let previousUploadedChunk = RemoteFileChunk( + fileName: String(previousUploadedChunkNum), + size: Int64(chunkSize), + remoteChunkStoreFolderName: uploadUuid + ) + let previousUploadedChunks = [previousUploadedChunk] + remoteInterface.currentChunks = [uploadUuid: previousUploadedChunks] + + let remotePath = account.davFilesUrl + "/file.txt" + var uploadedChunks = [RemoteFileChunk]() + let result = await NextcloudFileProviderKit.upload( + fileLocatedAt: fileUrl, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + inChunksSized: chunkSize, + usingChunkUploadId: uploadUuid, + dbManager: dbManager, + chunkUploadCompleteHandler: { uploadedChunks.append($0) } + ) + let resultChunks = try XCTUnwrap(result.chunks) + let expectedChunkCount = Int(ceil(Double(data.count) / Double(chunkSize))) + + XCTAssertEqual(result.remoteError, .success) + XCTAssertEqual(resultChunks.count, expectedChunkCount) + XCTAssertEqual(result.size, Int64(data.count)) + XCTAssertNotNil(result.ocId) + XCTAssertNotNil(result.etag) + + XCTAssertEqual(uploadedChunks.count, resultChunks.count - previousUploadedChunks.count) + + let firstUploadedChunk = try XCTUnwrap(uploadedChunks.first) + let firstUploadedChunkNameInt = try XCTUnwrap(Int(firstUploadedChunk.fileName)) + let lastUploadedChunk = try XCTUnwrap(uploadedChunks.last) + let lastUploadedChunkNameInt = try XCTUnwrap(Int(lastUploadedChunk.fileName)) + XCTAssertEqual(firstUploadedChunkNameInt, previousUploadedChunkNum + 1) + XCTAssertEqual(lastUploadedChunkNameInt, previousUploadedChunkNum + 2) + print(uploadedChunks) + XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) + XCTAssertEqual( + Int(lastUploadedChunk.size), data.count - ((lastUploadedChunkNameInt - 1) * chunkSize) + ) + } } From 9551c7b246fedddac6d41192100577ca7178f627 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:13:37 +0800 Subject: [PATCH 49/86] Fix expected chunk size in normal chunked upload test Signed-off-by: Claudio Cambra --- Tests/NextcloudFileProviderKitTests/UploadTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index 9383b0fe..831299ed 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -82,7 +82,7 @@ final class UploadTests: XCTestCase { XCTAssertEqual(lastUploadedChunkNameInt, expectedChunkCount) XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) XCTAssertEqual( - Int(lastUploadedChunk.size), data.count - (lastUploadedChunkNameInt * chunkSize) + Int(lastUploadedChunk.size), data.count - ((lastUploadedChunkNameInt - 1) * chunkSize) ) } From 488993d737b268002b1a6bdb43564c43a05e8972 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:14:21 +0800 Subject: [PATCH 50/86] Take fileLocatedAt in upload func as a string Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Utilities/Upload.swift | 12 ++++++------ .../NextcloudFileProviderKitTests/UploadTests.swift | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 2e1cdc01..239b0faf 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -15,7 +15,7 @@ let defaultFileChunkSize = 10_000_000 // 10 MB let uploadLogger = Logger(subsystem: Logger.subsystem, category: "upload") func upload( - fileLocatedAt localFileUrl: URL, + fileLocatedAt localFilePath: String, toRemotePath remotePath: String, usingRemoteInterface remoteInterface: RemoteInterface, withAccount account: Account, @@ -38,13 +38,12 @@ func upload( afError: AFError?, remoteError: NKError ) { - let localPath = localFileUrl.path let fileSize = - (try? FileManager.default.attributesOfItem(atPath: localPath)[.size] as? Int64) ?? 0 + (try? FileManager.default.attributesOfItem(atPath: localFilePath)[.size] as? Int64) ?? 0 guard fileSize > chunkSize else { let (_, ocId, etag, date, size, _, afError, remoteError) = await remoteInterface.upload( remotePath: remotePath, - localPath: localFileUrl.path, + localPath: localFilePath, creationDate: creationDate, modificationDate: modificationDate, account: account, @@ -65,8 +64,9 @@ func upload( ) } - let localFileName = localFileUrl.lastPathComponent - let localParentDirectoryPath = localFileUrl.deletingLastPathComponent().path + let localFilePathNs = localFilePath as NSString + let localFileName = localFilePathNs.lastPathComponent + let localParentDirectoryPath = localFilePathNs.deletingLastPathComponent let remoteParentDirectoryPath = (remotePath as NSString).deletingLastPathComponent as String let remainingChunks = dbManager diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index 831299ed..c39a05df 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -29,7 +29,7 @@ final class UploadTests: XCTestCase { MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: account)) let remotePath = account.davFilesUrl + "/file.txt" let result = await NextcloudFileProviderKit.upload( - fileLocatedAt: fileUrl, + fileLocatedAt: fileUrl.path, toRemotePath: remotePath, usingRemoteInterface: remoteInterface, withAccount: account, @@ -55,7 +55,7 @@ final class UploadTests: XCTestCase { let chunkSize = 3 var uploadedChunks = [RemoteFileChunk]() let result = await NextcloudFileProviderKit.upload( - fileLocatedAt: fileUrl, + fileLocatedAt: fileUrl.path, toRemotePath: remotePath, usingRemoteInterface: remoteInterface, withAccount: account, @@ -108,7 +108,7 @@ final class UploadTests: XCTestCase { let remotePath = account.davFilesUrl + "/file.txt" var uploadedChunks = [RemoteFileChunk]() let result = await NextcloudFileProviderKit.upload( - fileLocatedAt: fileUrl, + fileLocatedAt: fileUrl.path, toRemotePath: remotePath, usingRemoteInterface: remoteInterface, withAccount: account, From 090919dbf61d68f5c0bae55c9bc641e345ae3ef4 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:28:02 +0800 Subject: [PATCH 51/86] Convert chunk bool property in itemmetadata to a chunkUploadId Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift b/Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift index 2f414106..51dd3d94 100644 --- a/Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift +++ b/Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift @@ -44,7 +44,7 @@ public class ItemMetadata: Object { @Persisted public var account = "" @Persisted public var assetLocalIdentifier = "" @Persisted public var checksums = "" - @Persisted public var chunk: Bool = false + @Persisted public var chunkUploadId: String = "" @Persisted public var classFile = "" @Persisted public var commentsUnread: Bool = false @Persisted public var contentType = "" From cd59ca73b4b845c4ba4c72087b19cfa32de950b5 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:28:43 +0800 Subject: [PATCH 52/86] Apply appropriate chunkUploadId state when setting upload state in FilesDatabaseManager Signed-off-by: Claudio Cambra --- .../Database/FilesDatabaseManager.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift index 603ab211..ed5b6bf1 100644 --- a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift +++ b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift @@ -340,6 +340,9 @@ public class FilesDatabaseManager { result.downloaded = false } else if result.isUpload { result.uploaded = false + result.chunkUploadId = UUID().uuidString + } else if status == .normal { + result.chunkUploadId = "" } Self.logger.debug( From 13a0aeb84bf31de8e64f879f7b0e41ab670ccea2 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:31:09 +0800 Subject: [PATCH 53/86] Only reset chunkUploadId if old metadata state was upload Signed-off-by: Claudio Cambra --- .../Database/FilesDatabaseManager.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift index ed5b6bf1..2e9378c8 100644 --- a/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift +++ b/Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift @@ -341,7 +341,7 @@ public class FilesDatabaseManager { } else if result.isUpload { result.uploaded = false result.chunkUploadId = UUID().uuidString - } else if status == .normal { + } else if status == .normal, metadata.isUpload { result.chunkUploadId = "" } From e456a79f72b3594ec4c9dce2e8d42535e9b6705f Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:31:59 +0800 Subject: [PATCH 54/86] Use new upload function in new item file creation Signed-off-by: Claudio Cambra --- .../Item/Item+Create.swift | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift index b6565d6f..53c968fc 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift @@ -105,12 +105,15 @@ extension Item { progress: Progress, dbManager: FilesDatabaseManager ) async -> (Item?, Error?) { - let (_, ocId, etag, date, size, _, _, error) = await remoteInterface.upload( - remotePath: remotePath, - localPath: localPath, + let (ocId, _, etag, date, size, _, error) = await upload( + fileLocatedAt: localPath, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + usingChunkUploadId: itemTemplate.itemIdentifier.rawValue, + dbManager: dbManager, creationDate: itemTemplate.creationDate as? Date, modificationDate: itemTemplate.contentModificationDate as? Date, - account: account, options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in @@ -147,8 +150,8 @@ extension Item { filename: \(itemTemplate.filename, privacy: .public) ocId: \(ocId, privacy: .public) etag: \(etag ?? "", privacy: .public) - date: \(date ?? NSDate(), privacy: .public) - size: \(size, privacy: .public), + date: \(date ?? Date(), privacy: .public) + size: \(Int(size ?? -1), privacy: .public), account: \(account.ncKitAccount, privacy: .public) """ ) @@ -157,20 +160,20 @@ extension Item { Self.logger.warning( """ Created item upload reported as successful, but there are differences between - the received file size (\(size, privacy: .public)) + the received file size (\(Int(size ?? -1), privacy: .public)) and the original file size (\(itemTemplate.documentSize??.int64Value ?? 0)) """ ) } let newMetadata = ItemMetadata() - newMetadata.date = (date ?? NSDate()) as Date + newMetadata.date = date ?? Date() newMetadata.etag = etag ?? "" newMetadata.account = account.ncKitAccount newMetadata.fileName = itemTemplate.filename newMetadata.fileNameView = itemTemplate.filename newMetadata.ocId = ocId - newMetadata.size = size + newMetadata.size = size ?? 0 newMetadata.contentType = itemTemplate.contentType?.preferredMIMEType ?? "" newMetadata.directory = false newMetadata.serverUrl = parentItemRemotePath From fa91e01eee58c775f2e4df2653ef30fffba9a29b Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:32:14 +0800 Subject: [PATCH 55/86] Also use new upload function for bundle inner item creation Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Item/Item+Create.swift | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift index 53c968fc..80da9938 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift @@ -305,12 +305,14 @@ extension Item { Handling child bundle or package file at: \(childUrlPath, privacy: .public) """ ) - let (_, _, _, _, _, _, _, error) = await remoteInterface.upload( - remotePath: childRemoteUrl, - localPath: childUrlPath, + let (_, _, _, _, _, _, error) = await upload( + fileLocatedAt: childUrlPath, + toRemotePath: childRemoteUrl, + usingRemoteInterface: remoteInterface, + withAccount: account, + dbManager: dbManager, creationDate: childUrlAttributes.creationDate, modificationDate: childUrlAttributes.contentModificationDate, - account: account, options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in From 7edc3a8fb8c816f7a4a3c10ef4f2b6d563bd8fef Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:33:00 +0800 Subject: [PATCH 56/86] Use new upload function when uploading updated item files Signed-off-by: Claudio Cambra --- .../Item/Item+Modify.swift | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift index 576ef94a..651d1a16 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift @@ -132,12 +132,15 @@ public extension Item { ) } - let (_, _, etag, date, size, _, _, error) = await remoteInterface.upload( - remotePath: remotePath, - localPath: newContents.path, + let (_, _, etag, date, size, _, error) = await upload( + fileLocatedAt: newContents.path, + toRemotePath: remotePath, + usingRemoteInterface: remoteInterface, + withAccount: account, + usingChunkUploadId: metadata.chunkUploadId, + dbManager: dbManager, creationDate: newCreationDate, modificationDate: newContentModificationDate, - account: account, options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in @@ -180,23 +183,24 @@ public extension Item { Self.logger.warning( """ Item content modification upload reported as successful, - but there are differences between the received file size (\(size, privacy: .public)) + but there are differences between the received file size (\(size ?? -1, privacy: .public)) and the original file size (\(self.documentSize?.int64Value ?? 0)) """ ) } let newMetadata = ItemMetadata(value: metadata) - newMetadata.date = (date ?? NSDate()) as Date + newMetadata.date = date ?? Date() newMetadata.etag = etag ?? metadata.etag newMetadata.ocId = ocId - newMetadata.size = size + newMetadata.size = size ?? 0 newMetadata.session = "" newMetadata.sessionError = "" newMetadata.sessionTaskIdentifier = 0 newMetadata.status = ItemMetadata.Status.normal.rawValue newMetadata.downloaded = true newMetadata.uploaded = true + newMetadata.chunkUploadId = "" dbManager.addItemMetadata(newMetadata) From b6a80ec03a1ebd6ea854036277ba518cb9669aa0 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 11:33:14 +0800 Subject: [PATCH 57/86] Use new upload function to upload inner bundle files Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Item/Item+Modify.swift | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift index 651d1a16..c2038301 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift @@ -392,12 +392,14 @@ public extension Item { Handling child bundle or package file at: \(childUrlPath, privacy: .public) """ ) - let (_, _, _, _, _, _, _, error) = await remoteInterface.upload( - remotePath: childRemoteUrl, - localPath: childUrlPath, + let (_, _, _, _, _, _, error) = await upload( + fileLocatedAt: childUrlPath, + toRemotePath: childRemoteUrl, + usingRemoteInterface: remoteInterface, + withAccount: account, + dbManager: dbManager, creationDate: childUrlAttributes.creationDate, modificationDate: childUrlAttributes.contentModificationDate, - account: account, options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in From f8bbf889969148468cde502f8b9c3a40cb2e5208 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 12:30:05 +0800 Subject: [PATCH 58/86] Use URLs to do splicing during chunk upload procedure Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Utilities/Upload.swift | 12 ++++++++---- .../ItemUploadTests.swift | 0 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 Tests/NextcloudFileProviderKitTests/ItemUploadTests.swift diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 239b0faf..3d5305a1 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -64,10 +64,14 @@ func upload( ) } - let localFilePathNs = localFilePath as NSString - let localFileName = localFilePathNs.lastPathComponent - let localParentDirectoryPath = localFilePathNs.deletingLastPathComponent - let remoteParentDirectoryPath = (remotePath as NSString).deletingLastPathComponent as String + let localFileUrl = URL(fileURLWithPath: localFilePath) + let localFileName = localFileUrl.lastPathComponent + let localParentDirectoryPath = localFileUrl.deletingLastPathComponent().path + guard let remoteUrl = URL(string: remotePath) else { + uploadLogger.error("Invalid remote path: \(remotePath, privacy: .public)") + return (nil, nil, nil, nil, nil, nil, .urlError) + } + let remoteParentDirectoryPath = remoteUrl.deletingLastPathComponent().absoluteString let remainingChunks = dbManager .ncDatabase() diff --git a/Tests/NextcloudFileProviderKitTests/ItemUploadTests.swift b/Tests/NextcloudFileProviderKitTests/ItemUploadTests.swift new file mode 100644 index 00000000..e69de29b From dbb92755da19cd582a4fa54d693b301a873e4521 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 12:30:16 +0800 Subject: [PATCH 59/86] Actually return ocId post chunk upload Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Utilities/Upload.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 3d5305a1..47247951 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -139,7 +139,7 @@ func upload( ) return ( - ocId: file?.name, + ocId: file?.ocId, chunks: chunks, etag: file?.etag, date: file?.date, From d33b75c5f68d1ad377fc18dbd5d54b89395c132e Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 12:30:35 +0800 Subject: [PATCH 60/86] Add test for chunked file create upload Signed-off-by: Claudio Cambra --- .../ItemCreateTests.swift | 54 ++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift index cc272647..59001766 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift @@ -18,12 +18,13 @@ final class ItemCreateTests: XCTestCase { user: "testUser", id: "testUserId", serverUrl: "https://mock.nc.com", password: "abcd" ) - lazy var rootItem = MockRemoteItem.rootItem(account: Self.account) + var rootItem: MockRemoteItem! static let dbManager = FilesDatabaseManager(realmConfig: .defaultConfiguration) override func setUp() { super.setUp() Realm.Configuration.defaultConfiguration.inMemoryIdentifier = name + rootItem = MockRemoteItem.rootItem(account: Self.account) } override func tearDown() { @@ -347,4 +348,55 @@ final class ItemCreateTests: XCTestCase { let childrenCount = Self.dbManager.childItemCount(directoryMetadata: dbItem) XCTAssertEqual(childrenCount, 6) // Ensure all children recorded to database } + + func testCreateFileChunked() async throws { + let remoteInterface = MockRemoteInterface(rootItem: rootItem) + let fileItemMetadata = ItemMetadata() + fileItemMetadata.fileName = "file" + fileItemMetadata.fileNameView = "file" + fileItemMetadata.directory = false + fileItemMetadata.classFile = NKCommon.TypeClassFile.document.rawValue + fileItemMetadata.serverUrl = Self.account.davFilesUrl + + let tempUrl = FileManager.default.temporaryDirectory.appendingPathComponent("file") + let tempData = Data(repeating: 1, count: defaultFileChunkSize * 3) + try Data(repeating: 1, count: defaultFileChunkSize * 3).write(to: tempUrl) + + let fileItemTemplate = Item( + metadata: fileItemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + let (createdItemMaybe, error) = await Item.create( + basedOn: fileItemTemplate, + contents: tempUrl, + account: Self.account, + remoteInterface: remoteInterface, + progress: Progress(), + dbManager: Self.dbManager + ) + let createdItem = try XCTUnwrap(createdItemMaybe) + + XCTAssertNil(error) + XCTAssertNotNil(createdItem) + XCTAssertEqual(createdItem.metadata.fileName, fileItemMetadata.fileName) + XCTAssertEqual(createdItem.metadata.directory, fileItemMetadata.directory) + + let remoteItem = try XCTUnwrap( + rootItem.children.first { $0.identifier == createdItem.itemIdentifier.rawValue } + ) + XCTAssertEqual(remoteItem.name, fileItemMetadata.fileName) + XCTAssertEqual(remoteItem.directory, fileItemMetadata.directory) + XCTAssertEqual(remoteItem.data, tempData) + + let dbItem = try XCTUnwrap( + Self.dbManager.itemMetadata(ocId: createdItem.itemIdentifier.rawValue) + ) + XCTAssertEqual(dbItem.fileName, fileItemMetadata.fileName) + XCTAssertEqual(dbItem.fileNameView, fileItemMetadata.fileNameView) + XCTAssertEqual(dbItem.directory, fileItemMetadata.directory) + XCTAssertEqual(dbItem.serverUrl, fileItemMetadata.serverUrl) + XCTAssertEqual(dbItem.ocId, createdItem.itemIdentifier.rawValue) + } } From bf68205ffd1e70af8c5a5f3b36984f5268c3c784 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 13:57:27 +0800 Subject: [PATCH 61/86] Do not double generate test data for chunked item creation Signed-off-by: Claudio Cambra --- Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift index 59001766..9e2451ac 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift @@ -360,7 +360,7 @@ final class ItemCreateTests: XCTestCase { let tempUrl = FileManager.default.temporaryDirectory.appendingPathComponent("file") let tempData = Data(repeating: 1, count: defaultFileChunkSize * 3) - try Data(repeating: 1, count: defaultFileChunkSize * 3).write(to: tempUrl) + try tempData.write(to: tempUrl) let fileItemTemplate = Item( metadata: fileItemMetadata, From 6d5fa1b043d49b41d2704cbf376def5914e813d6 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 13:58:01 +0800 Subject: [PATCH 62/86] Track expected transfer size of completed chunked uploads, not counting present chunks, in MRI Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 33405e5d..65f5e4da 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -18,6 +18,7 @@ public class MockRemoteInterface: RemoteInterface { public var delegate: (any NextcloudKitDelegate)? public var rootTrashItem: MockRemoteItem? public var currentChunks: [String: [RemoteFileChunk]] = [:] + public var completedChunkTransferSize: [String: Int64] = [:] public init(rootItem: MockRemoteItem? = nil, rootTrashItem: MockRemoteItem? = nil) { self.rootItem = rootItem @@ -290,6 +291,8 @@ public class MockRemoteInterface: RemoteInterface { chunkUploadCompleteHandler(chunk) } } + completedChunkTransferSize[remoteChunkStoreFolderName] = + size - preexistingChunks.reduce(0) { $0 + $1.size } let file = NKFile() file.fileName = localFileName From 0ea093173e6dc9c183acf23220f35032271bcd7f Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 13:58:21 +0800 Subject: [PATCH 63/86] Add test for resumed chunked upload of newly created item Signed-off-by: Claudio Cambra --- .../ItemCreateTests.swift | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift index 9e2451ac..fb9792cc 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift @@ -399,4 +399,81 @@ final class ItemCreateTests: XCTestCase { XCTAssertEqual(dbItem.serverUrl, fileItemMetadata.serverUrl) XCTAssertEqual(dbItem.ocId, createdItem.itemIdentifier.rawValue) } + + func testCreateFileChunkedResumed() async throws { + let chunkUploadId = UUID().uuidString + let preexistingChunk = RemoteFileChunk( + fileName: String(1), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ) + let db = Self.dbManager.ncDatabase() + try db.write { db.add(preexistingChunk) } + + let remoteInterface = MockRemoteInterface(rootItem: rootItem) + remoteInterface.currentChunks = [chunkUploadId: [preexistingChunk]] + + // With real new item uploads we do not have an associated ItemMetadata as the template is + // passed onto us by the OS. We cannot rely on the chunkUploadId property we usually use + // during modified item uploads. + // + // We therefore can only use the system-provided item template's itemIdentifier as the + // chunked upload identifier during new item creation. + // + // To test this situation we set the ocId of the metadata used to construct the item + // template to the chunk upload id. + let fileItemMetadata = ItemMetadata() + fileItemMetadata.ocId = chunkUploadId + fileItemMetadata.fileName = "file" + fileItemMetadata.fileNameView = "file" + fileItemMetadata.directory = false + fileItemMetadata.classFile = NKCommon.TypeClassFile.document.rawValue + fileItemMetadata.serverUrl = Self.account.davFilesUrl + + let tempUrl = FileManager.default.temporaryDirectory.appendingPathComponent("file") + let tempData = Data(repeating: 1, count: defaultFileChunkSize * 3) + try tempData.write(to: tempUrl) + + let fileItemTemplate = Item( + metadata: fileItemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + let (createdItemMaybe, error) = await Item.create( + basedOn: fileItemTemplate, + contents: tempUrl, + account: Self.account, + remoteInterface: remoteInterface, + progress: Progress(), + dbManager: Self.dbManager + ) + let createdItem = try XCTUnwrap(createdItemMaybe) + + XCTAssertNil(error) + XCTAssertNotNil(createdItem) + XCTAssertEqual(createdItem.metadata.fileName, fileItemMetadata.fileName) + XCTAssertEqual(createdItem.metadata.directory, fileItemMetadata.directory) + + let remoteItem = try XCTUnwrap( + rootItem.children.first { $0.identifier == createdItem.itemIdentifier.rawValue } + ) + XCTAssertEqual(remoteItem.name, fileItemMetadata.fileName) + XCTAssertEqual(remoteItem.directory, fileItemMetadata.directory) + XCTAssertEqual(remoteItem.data, tempData) + XCTAssertEqual( + remoteInterface.completedChunkTransferSize[chunkUploadId], + Int64(tempData.count) - preexistingChunk.size + ) + + let dbItem = try XCTUnwrap( + Self.dbManager.itemMetadata(ocId: createdItem.itemIdentifier.rawValue) + ) + XCTAssertEqual(dbItem.fileName, fileItemMetadata.fileName) + XCTAssertEqual(dbItem.fileNameView, fileItemMetadata.fileNameView) + XCTAssertEqual(dbItem.directory, fileItemMetadata.directory) + XCTAssertEqual(dbItem.serverUrl, fileItemMetadata.serverUrl) + XCTAssertEqual(dbItem.ocId, createdItem.itemIdentifier.rawValue) + XCTAssertTrue(dbItem.chunkUploadId.isEmpty) + } } From 176610475a20866a0f4564d10bf418f3a38e0b5a Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:02:03 +0800 Subject: [PATCH 64/86] Modify RemoteInterface to directly take a local path and a remote path for chunked uploads This makes the API sensible and consistent with the default upload method used in NCFPK. Note that this requires upstream changes in NextcloudKit: https://github.com/nextcloud/NextcloudKit/pull/116 Signed-off-by: Claudio Cambra --- .../NextcloudKit+RemoteInterface.swift | 18 ++++++++++----- .../Interface/RemoteInterface.swift | 5 ++-- .../Utilities/Upload.swift | 18 ++++----------- Tests/Interface/MockRemoteInterface.swift | 23 +++++++++++-------- 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 1b5cd765..9f2e5c75 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -81,9 +81,8 @@ extension NextcloudKit: RemoteInterface { } public func chunkedUpload( - localDirectoryPath: String, - localFileName: String, - remoteParentDirectoryPath: String, + localPath: String, + remotePath: String, remoteChunkStoreFolderName: String = UUID().uuidString, chunkSize: Int, remainingChunks: [RemoteFileChunk], @@ -105,13 +104,20 @@ extension NextcloudKit: RemoteInterface { afError: AFError?, remoteError: NKError ) { + guard let remoteUrl = URL(string: remotePath) else { + uploadLogger.error("NCKit ext: Could not get url from \(remotePath, privacy: .public)") + return ("", nil, nil, nil, .urlError) + } + let localUrl = URL(fileURLWithPath: localPath) + return await withCheckedContinuation { continuation in uploadChunk( - directory: localDirectoryPath, - fileName: localFileName, + directory: localUrl.deletingLastPathComponent().path, + fileName: localUrl.lastPathComponent, + destinationFileName: remoteUrl.lastPathComponent, date: modificationDate, creationDate: creationDate, - serverUrl: remoteParentDirectoryPath, + serverUrl: remoteUrl.deletingLastPathComponent().absoluteString, chunkFolder: remoteChunkStoreFolderName, filesChunk: remainingChunks.toNcKitChunks(), chunkSize: chunkSize, diff --git a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift index 81e0719c..8c5d47f5 100644 --- a/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift @@ -53,9 +53,8 @@ public protocol RemoteInterface { ) func chunkedUpload( - localDirectoryPath: String, - localFileName: String, - remoteParentDirectoryPath: String, + localPath: String, + remotePath: String, remoteChunkStoreFolderName: String, chunkSize: Int, remainingChunks: [RemoteFileChunk], diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 47247951..8d544926 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -64,15 +64,6 @@ func upload( ) } - let localFileUrl = URL(fileURLWithPath: localFilePath) - let localFileName = localFileUrl.lastPathComponent - let localParentDirectoryPath = localFileUrl.deletingLastPathComponent().path - guard let remoteUrl = URL(string: remotePath) else { - uploadLogger.error("Invalid remote path: \(remotePath, privacy: .public)") - return (nil, nil, nil, nil, nil, nil, .urlError) - } - let remoteParentDirectoryPath = remoteUrl.deletingLastPathComponent().absoluteString - let remainingChunks = dbManager .ncDatabase() .objects(RemoteFileChunk.self) @@ -80,9 +71,8 @@ func upload( .filter { $0.remoteChunkStoreFolderName == chunkUploadId } let (_, chunks, file, afError, remoteError) = await remoteInterface.chunkedUpload( - localDirectoryPath: localParentDirectoryPath, - localFileName: localFileName, - remoteParentDirectoryPath: remoteParentDirectoryPath, + localPath: localFilePath, + remotePath: remotePath, remoteChunkStoreFolderName: chunkUploadId, chunkSize: chunkSize, remainingChunks: remainingChunks, @@ -94,12 +84,12 @@ func upload( chunkCounter: { currentChunk in uploadLogger.info( """ - \(localFileName, privacy: .public) current chunk: \(currentChunk, privacy: .public) + \(localFilePath, privacy: .public) current chunk: \(currentChunk, privacy: .public) """ ) }, chunkUploadStartHandler: { chunks in - uploadLogger.info("\(localFileName, privacy: .public) uploading chunk") + uploadLogger.info("\(localFilePath, privacy: .public) uploading chunk") let db = dbManager.ncDatabase() do { try db.write { db.add(chunks.map { RemoteFileChunk(value: $0) }) } diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index 65f5e4da..d09b843d 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -228,9 +228,8 @@ public class MockRemoteInterface: RemoteInterface { } public func chunkedUpload( - localDirectoryPath: String, - localFileName: String, - remoteParentDirectoryPath: String, + localPath: String, + remotePath: String, remoteChunkStoreFolderName: String, chunkSize: Int, remainingChunks: [RemoteFileChunk], @@ -252,14 +251,18 @@ public class MockRemoteInterface: RemoteInterface { afError: AFError?, remoteError: NKError ) { + guard let remoteUrl = URL(string: remotePath) else { + print("Invalid remote path!") + return ("", nil, nil, nil, .urlError) + } + // Create temp directory for file and create chunks within it let fm = FileManager.default let tempDirectoryUrl = fm.temporaryDirectory.appendingPathComponent(UUID().uuidString) try! fm.createDirectory(atPath: tempDirectoryUrl.path, withIntermediateDirectories: true) // Access local file and gather metadata - let wholeLocalFile = localDirectoryPath + "/" + localFileName - let fileSize = try! fm.attributesOfItem(atPath: wholeLocalFile)[.size] as! Int + let fileSize = try! fm.attributesOfItem(atPath: localPath)[.size] as! Int var remainingFileSize = fileSize let numChunks = Int(ceil(Double(fileSize) / Double(chunkSize))) @@ -276,8 +279,8 @@ public class MockRemoteInterface: RemoteInterface { chunkUploadStartHandler(chunks) let (_, ocId, etag, date, size, _, afError, remoteError) = await upload( - remotePath: remoteParentDirectoryPath + "/" + localFileName, - localPath: wholeLocalFile, + remotePath: remotePath, + localPath: localPath, creationDate: creationDate, modificationDate: modificationDate, account: account, @@ -295,12 +298,12 @@ public class MockRemoteInterface: RemoteInterface { size - preexistingChunks.reduce(0) { $0 + $1.size } let file = NKFile() - file.fileName = localFileName + file.fileName = remoteUrl.lastPathComponent file.etag = etag ?? "" file.size = size - file.path = remoteParentDirectoryPath + "/" + localFileName + file.path = remotePath file.ocId = ocId ?? "" - file.serverUrl = remoteParentDirectoryPath + file.serverUrl = remoteUrl.deletingLastPathComponent().absoluteString file.urlBase = account.serverUrl file.user = account.username file.userId = account.id From effa38c49abf9063dc8bb835b354af26c2287bf3 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:02:36 +0800 Subject: [PATCH 65/86] Delete unused ItemUploadTests Signed-off-by: Claudio Cambra --- Tests/NextcloudFileProviderKitTests/ItemUploadTests.swift | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 Tests/NextcloudFileProviderKitTests/ItemUploadTests.swift diff --git a/Tests/NextcloudFileProviderKitTests/ItemUploadTests.swift b/Tests/NextcloudFileProviderKitTests/ItemUploadTests.swift deleted file mode 100644 index e69de29b..00000000 From 258c998a09c918a1053d9a3d47dce05dacfcfd89 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:03:32 +0800 Subject: [PATCH 66/86] Add a test for chunked upload of modified item contents Signed-off-by: Claudio Cambra --- .../ItemModifyTests.swift | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift index 6f5dda5b..d92b3f33 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift @@ -1205,4 +1205,49 @@ final class ItemModifyTests: XCTestCase { ) XCTAssertEqual(untrashedFolderChildItem.serverUrl, remoteTrashFolder.remotePath) } + + func testModifyFileContentsChunked() async throws { + let remoteInterface = MockRemoteInterface(rootItem: rootItem) + + let itemMetadata = remoteItem.toItemMetadata(account: Self.account) + Self.dbManager.addItemMetadata(itemMetadata) + + let newContents = Data(repeating: 1, count: defaultFileChunkSize * 3) + let newContentsUrl = FileManager.default.temporaryDirectory.appendingPathComponent("test") + try newContents.write(to: newContentsUrl) + + let targetItemMetadata = ItemMetadata(value: itemMetadata) + targetItemMetadata.date = .init() + targetItemMetadata.size = Int64(newContents.count) + + let item = Item( + metadata: itemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + + let targetItem = Item( + metadata: targetItemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + targetItem.dbManager = Self.dbManager + + let (modifiedItemMaybe, error) = await item.modify( + itemTarget: targetItem, + changedFields: [.contents, .contentModificationDate], + contents: newContentsUrl, + dbManager: Self.dbManager + ) + XCTAssertNil(error) + let modifiedItem = try XCTUnwrap(modifiedItemMaybe) + + XCTAssertEqual(modifiedItem.itemIdentifier, targetItem.itemIdentifier) + XCTAssertEqual(modifiedItem.contentModificationDate, targetItem.contentModificationDate) + XCTAssertEqual(modifiedItem.documentSize?.intValue, newContents.count) + + XCTAssertEqual(remoteItem.data, newContents) + } } From 7505ee8f4d60f81eafcda472bf86d263735e61f1 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:24:41 +0800 Subject: [PATCH 67/86] Add test for modified item chunked upload, resumed Signed-off-by: Claudio Cambra --- .../ItemModifyTests.swift | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift index d92b3f33..297aeabb 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift @@ -1250,4 +1250,67 @@ final class ItemModifyTests: XCTestCase { XCTAssertEqual(remoteItem.data, newContents) } + + func testModifyFileContentsChunkedResumed() async throws { + let chunkUploadId = UUID().uuidString + let preexistingChunk = RemoteFileChunk( + fileName: String(1), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ) + let db = Self.dbManager.ncDatabase() + try db.write { db.add(preexistingChunk) } + + let remoteInterface = MockRemoteInterface(rootItem: rootItem) + remoteInterface.currentChunks = [chunkUploadId: [preexistingChunk]] + + let itemMetadata = remoteItem.toItemMetadata(account: Self.account) + itemMetadata.chunkUploadId = chunkUploadId + Self.dbManager.addItemMetadata(itemMetadata) + + let newContents = Data(repeating: 1, count: defaultFileChunkSize * 3) + let newContentsUrl = FileManager.default.temporaryDirectory.appendingPathComponent("test") + try newContents.write(to: newContentsUrl) + + let targetItemMetadata = ItemMetadata(value: itemMetadata) + targetItemMetadata.date = .init() + targetItemMetadata.size = Int64(newContents.count) + + let item = Item( + metadata: itemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + + let targetItem = Item( + metadata: targetItemMetadata, + parentItemIdentifier: .rootContainer, + account: Self.account, + remoteInterface: remoteInterface + ) + targetItem.dbManager = Self.dbManager + + let (modifiedItemMaybe, error) = await item.modify( + itemTarget: targetItem, + changedFields: [.contents, .contentModificationDate], + contents: newContentsUrl, + dbManager: Self.dbManager + ) + XCTAssertNil(error) + let modifiedItem = try XCTUnwrap(modifiedItemMaybe) + + XCTAssertEqual(modifiedItem.itemIdentifier, targetItem.itemIdentifier) + XCTAssertEqual(modifiedItem.contentModificationDate, targetItem.contentModificationDate) + XCTAssertEqual(modifiedItem.documentSize?.intValue, newContents.count) + + XCTAssertEqual(remoteItem.data, newContents) + XCTAssertEqual( + remoteInterface.completedChunkTransferSize[chunkUploadId], + Int64(newContents.count) - preexistingChunk.size + ) + + let dbItem = try XCTUnwrap(Self.dbManager.itemMetadata(ocId: itemMetadata.ocId)) + XCTAssertTrue(dbItem.chunkUploadId.isEmpty) + } } From 379513ac9f48a9d329dd027deb5790eb0e2e6c54 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:57:59 +0800 Subject: [PATCH 68/86] Properly employ remainingChunks argument in mock remote interface Signed-off-by: Claudio Cambra --- Tests/Interface/MockRemoteInterface.swift | 34 +++++++++++------------ 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Tests/Interface/MockRemoteInterface.swift b/Tests/Interface/MockRemoteInterface.swift index d09b843d..ca1f8d53 100644 --- a/Tests/Interface/MockRemoteInterface.swift +++ b/Tests/Interface/MockRemoteInterface.swift @@ -266,17 +266,20 @@ public class MockRemoteInterface: RemoteInterface { var remainingFileSize = fileSize let numChunks = Int(ceil(Double(fileSize) / Double(chunkSize))) - let chunks = (0.. Date: Wed, 8 Jan 2025 15:58:28 +0800 Subject: [PATCH 69/86] Add expected remaining chunks to database in upload tests for resumed chunked upload Signed-off-by: Claudio Cambra --- .../UploadTests.swift | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Tests/NextcloudFileProviderKitTests/UploadTests.swift b/Tests/NextcloudFileProviderKitTests/UploadTests.swift index c39a05df..a5d5a499 100644 --- a/Tests/NextcloudFileProviderKitTests/UploadTests.swift +++ b/Tests/NextcloudFileProviderKitTests/UploadTests.swift @@ -105,6 +105,22 @@ final class UploadTests: XCTestCase { let previousUploadedChunks = [previousUploadedChunk] remoteInterface.currentChunks = [uploadUuid: previousUploadedChunks] + let db = dbManager.ncDatabase() + try db.write { + db.add([ + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 1), + size: Int64(chunkSize), + remoteChunkStoreFolderName: uploadUuid + ), + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 2), + size: Int64(data.count - (chunkSize * (previousUploadedChunkNum + 1))), + remoteChunkStoreFolderName: uploadUuid + ) + ]) + } + let remotePath = account.davFilesUrl + "/file.txt" var uploadedChunks = [RemoteFileChunk]() let result = await NextcloudFileProviderKit.upload( From 566ac133d34832b3a7f2cac7a9fa9eae8e39e388 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:58:49 +0800 Subject: [PATCH 70/86] Fix state of remaining chunks in database in resumed chunked upload of modified item contents Signed-off-by: Claudio Cambra --- .../ItemModifyTests.swift | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift index 297aeabb..ad008eca 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemModifyTests.swift @@ -1253,13 +1253,28 @@ final class ItemModifyTests: XCTestCase { func testModifyFileContentsChunkedResumed() async throws { let chunkUploadId = UUID().uuidString + let previousUploadedChunkNum = 1 let preexistingChunk = RemoteFileChunk( - fileName: String(1), + fileName: String(previousUploadedChunkNum), size: Int64(defaultFileChunkSize), remoteChunkStoreFolderName: chunkUploadId ) + let db = Self.dbManager.ncDatabase() - try db.write { db.add(preexistingChunk) } + try db.write { + db.add([ + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 1), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ), + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 2), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ) + ]) + } let remoteInterface = MockRemoteInterface(rootItem: rootItem) remoteInterface.currentChunks = [chunkUploadId: [preexistingChunk]] From bc2e5ea85fdcbf5c4d497eca12815ec0012fe475 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:59:07 +0800 Subject: [PATCH 71/86] Fix state of remaining chunks in database in resumed chunked upload of newly created file Signed-off-by: Claudio Cambra --- .../ItemCreateTests.swift | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift index fb9792cc..540325eb 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift @@ -402,13 +402,28 @@ final class ItemCreateTests: XCTestCase { func testCreateFileChunkedResumed() async throws { let chunkUploadId = UUID().uuidString + let previousUploadedChunkNum = 1 let preexistingChunk = RemoteFileChunk( - fileName: String(1), + fileName: String(previousUploadedChunkNum), size: Int64(defaultFileChunkSize), remoteChunkStoreFolderName: chunkUploadId ) + let db = Self.dbManager.ncDatabase() - try db.write { db.add(preexistingChunk) } + try db.write { + db.add([ + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 1), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ), + RemoteFileChunk( + fileName: String(previousUploadedChunkNum + 2), + size: Int64(defaultFileChunkSize), + remoteChunkStoreFolderName: chunkUploadId + ) + ]) + } let remoteInterface = MockRemoteInterface(rootItem: rootItem) remoteInterface.currentChunks = [chunkUploadId: [preexistingChunk]] From 916b34db00f3aa3572ee98e210f7708112fe7a57 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:59:24 +0800 Subject: [PATCH 72/86] Add test for chunked upload of MRI Signed-off-by: Claudio Cambra --- .../MockRemoteInterfaceTests.swift | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/Tests/InterfaceTests/MockRemoteInterfaceTests.swift b/Tests/InterfaceTests/MockRemoteInterfaceTests.swift index 0a139ca9..91c6a864 100644 --- a/Tests/InterfaceTests/MockRemoteInterfaceTests.swift +++ b/Tests/InterfaceTests/MockRemoteInterfaceTests.swift @@ -168,6 +168,122 @@ final class MockRemoteInterfaceTests: XCTestCase { // TODO: Add test for overwriting existing file } + func testChunkedUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: Self.account)) + let remotePath = Self.account.davFilesUrl + "/file.txt" + let chunkSize = 3 + var uploadedChunks = [RemoteFileChunk]() + let result = await remoteInterface.chunkedUpload( + localPath: fileUrl.path, + remotePath: remotePath, + remoteChunkStoreFolderName: UUID().uuidString, + chunkSize: chunkSize, + remainingChunks: [], + creationDate: .init(), + modificationDate: .init(), + account: Self.account, + options: .init(), + chunkUploadCompleteHandler: { uploadedChunks.append($0) } + ) + + let resultChunks = try XCTUnwrap(result.fileChunks) + let expectedChunkCount = Int(ceil(Double(data.count) / Double(chunkSize))) + + XCTAssertEqual(result.remoteError, .success) + XCTAssertEqual(resultChunks.count, expectedChunkCount) + XCTAssertNotNil(result.file) + XCTAssertEqual(result.file?.size, Int64(data.count)) + + XCTAssertEqual(uploadedChunks.count, resultChunks.count) + + let firstUploadedChunk = try XCTUnwrap(uploadedChunks.first) + let firstUploadedChunkNameInt = try XCTUnwrap(Int(firstUploadedChunk.fileName)) + let lastUploadedChunk = try XCTUnwrap(uploadedChunks.last) + let lastUploadedChunkNameInt = try XCTUnwrap(Int(lastUploadedChunk.fileName)) + XCTAssertEqual(firstUploadedChunkNameInt, 1) + XCTAssertEqual(lastUploadedChunkNameInt, expectedChunkCount) + XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) + XCTAssertEqual( + Int(lastUploadedChunk.size), data.count - ((lastUploadedChunkNameInt - 1) * chunkSize) + ) + } + + func testResumedChunkedUpload() async throws { + let fileUrl = + FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let data = Data(repeating: 1, count: 8) + try data.write(to: fileUrl) + + let chunkSize = 3 + let uploadUuid = UUID().uuidString + let previousUploadedChunkNum = 1 + let previousUploadedChunk = RemoteFileChunk( + fileName: String(previousUploadedChunkNum), + size: Int64(chunkSize), + remoteChunkStoreFolderName: uploadUuid + ) + let previousUploadedChunks = [previousUploadedChunk] + + let remoteInterface = + MockRemoteInterface(rootItem: MockRemoteItem.rootItem(account: Self.account)) + remoteInterface.currentChunks = [uploadUuid: previousUploadedChunks] + + let remotePath = Self.account.davFilesUrl + "/file.txt" + + var uploadedChunks = [RemoteFileChunk]() + let result = await remoteInterface.chunkedUpload( + localPath: fileUrl.path, + remotePath: remotePath, + remoteChunkStoreFolderName: uploadUuid, + chunkSize: chunkSize, + remainingChunks: [ + RemoteFileChunk( + fileName: String(2), + size: Int64(chunkSize), + remoteChunkStoreFolderName: uploadUuid + ), + RemoteFileChunk( + fileName: String(3), + size: Int64(data.count - (chunkSize * 2)), + remoteChunkStoreFolderName: uploadUuid + ) + ], + creationDate: .init(), + modificationDate: .init(), + account: Self.account, + options: .init(), + chunkUploadCompleteHandler: { uploadedChunks.append($0) } + ) + + let resultChunks = try XCTUnwrap(result.fileChunks) + let expectedChunkCount = Int(ceil(Double(data.count) / Double(chunkSize))) + + XCTAssertEqual(result.remoteError, .success) + XCTAssertEqual(resultChunks.count, expectedChunkCount) + XCTAssertNotNil(result.file) + XCTAssertEqual(result.file?.size, Int64(data.count)) + + XCTAssertEqual(uploadedChunks.count, resultChunks.count - previousUploadedChunks.count) + + let firstUploadedChunk = try XCTUnwrap(uploadedChunks.first) + let firstUploadedChunkNameInt = try XCTUnwrap(Int(firstUploadedChunk.fileName)) + let lastUploadedChunk = try XCTUnwrap(uploadedChunks.last) + let lastUploadedChunkNameInt = try XCTUnwrap(Int(lastUploadedChunk.fileName)) + XCTAssertEqual(firstUploadedChunkNameInt, previousUploadedChunkNum + 1) + XCTAssertEqual(lastUploadedChunkNameInt, previousUploadedChunkNum + 2) + print(uploadedChunks) + XCTAssertEqual(Int(firstUploadedChunk.size), chunkSize) + XCTAssertEqual( + Int(lastUploadedChunk.size), data.count - ((lastUploadedChunkNameInt - 1) * chunkSize) + ) + } + func testMove() async { let remoteInterface = MockRemoteInterface(rootItem: rootItem) let itemA = MockRemoteItem( From 76bd2e11546587885720a90c9b86985bdf9ad2cc Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 15:59:45 +0800 Subject: [PATCH 73/86] Temporarily use nckit branch with chunk destination changes Signed-off-by: Claudio Cambra --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 7926455d..288d5c88 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( url: "https://github.com/claucambra/NextcloudCapabilitiesKit.git", .upToNextMajor(from: "2.0.0") ), - .package(url: "https://github.com/nextcloud/NextcloudKit", branch: "develop"), + .package(url: "https://github.com/nextcloud/NextcloudKit", branch: "work/chunk-destination-filename"), .package(url: "https://github.com/realm/realm-swift.git", exact: "10.49.0"), .package(url: "https://github.com/apple/swift-nio.git", from: "2.0.0") ], From 704428a0aafb326c11795301964da7c316583c74 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 16:49:31 +0800 Subject: [PATCH 74/86] Add some logging for chunked upload Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Utilities/Upload.swift | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 8d544926..495fa21c 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -64,6 +64,15 @@ func upload( ) } + uploadLogger.info( + """ + Performing chunked upload to \(remotePath, privacy: .public) + localFilePath: \(localFilePath, privacy: .public) + remoteChunkStoreFolderName: \(chunkUploadId, privacy: .public). + chunkSize: \(chunkSize, privacy: .public) + """ + ) + let remainingChunks = dbManager .ncDatabase() .objects(RemoteFileChunk.self) From cceffe142c8d0df7b5058e635a85b187ce0a660e Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 17:04:48 +0800 Subject: [PATCH 75/86] Ensure we remove illegal chars from file provider item id used as chunk upload id Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Item/Item+Create.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift index 80da9938..1b3133c0 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift @@ -105,12 +105,14 @@ extension Item { progress: Progress, dbManager: FilesDatabaseManager ) async -> (Item?, Error?) { + let chunkUploadId = + itemTemplate.itemIdentifier.rawValue.replacingOccurrences(of: "/", with: "") let (ocId, _, etag, date, size, _, error) = await upload( fileLocatedAt: localPath, toRemotePath: remotePath, usingRemoteInterface: remoteInterface, withAccount: account, - usingChunkUploadId: itemTemplate.itemIdentifier.rawValue, + usingChunkUploadId: chunkUploadId, dbManager: dbManager, creationDate: itemTemplate.creationDate as? Date, modificationDate: itemTemplate.contentModificationDate as? Date, From e5503e2f0754506f928f6fb46da75c0e9dae7f81 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 17:17:04 +0800 Subject: [PATCH 76/86] Update item create test with chunked upload to check handling of illegal chunk upload ID Signed-off-by: Claudio Cambra --- .../ItemCreateTests.swift | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift index 540325eb..a8c7e8e4 100644 --- a/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift +++ b/Tests/NextcloudFileProviderKitTests/ItemCreateTests.swift @@ -401,12 +401,13 @@ final class ItemCreateTests: XCTestCase { } func testCreateFileChunkedResumed() async throws { - let chunkUploadId = UUID().uuidString + let expectedChunkUploadId = UUID().uuidString // Check if illegal characters are stripped + let illegalChunkUploadId = expectedChunkUploadId + "/" // Check if illegal characters are stripped let previousUploadedChunkNum = 1 let preexistingChunk = RemoteFileChunk( fileName: String(previousUploadedChunkNum), size: Int64(defaultFileChunkSize), - remoteChunkStoreFolderName: chunkUploadId + remoteChunkStoreFolderName: expectedChunkUploadId ) let db = Self.dbManager.ncDatabase() @@ -415,18 +416,18 @@ final class ItemCreateTests: XCTestCase { RemoteFileChunk( fileName: String(previousUploadedChunkNum + 1), size: Int64(defaultFileChunkSize), - remoteChunkStoreFolderName: chunkUploadId + remoteChunkStoreFolderName: expectedChunkUploadId ), RemoteFileChunk( fileName: String(previousUploadedChunkNum + 2), size: Int64(defaultFileChunkSize), - remoteChunkStoreFolderName: chunkUploadId + remoteChunkStoreFolderName: expectedChunkUploadId ) ]) } let remoteInterface = MockRemoteInterface(rootItem: rootItem) - remoteInterface.currentChunks = [chunkUploadId: [preexistingChunk]] + remoteInterface.currentChunks = [expectedChunkUploadId: [preexistingChunk]] // With real new item uploads we do not have an associated ItemMetadata as the template is // passed onto us by the OS. We cannot rely on the chunkUploadId property we usually use @@ -438,7 +439,7 @@ final class ItemCreateTests: XCTestCase { // To test this situation we set the ocId of the metadata used to construct the item // template to the chunk upload id. let fileItemMetadata = ItemMetadata() - fileItemMetadata.ocId = chunkUploadId + fileItemMetadata.ocId = illegalChunkUploadId fileItemMetadata.fileName = "file" fileItemMetadata.fileNameView = "file" fileItemMetadata.directory = false @@ -477,7 +478,7 @@ final class ItemCreateTests: XCTestCase { XCTAssertEqual(remoteItem.directory, fileItemMetadata.directory) XCTAssertEqual(remoteItem.data, tempData) XCTAssertEqual( - remoteInterface.completedChunkTransferSize[chunkUploadId], + remoteInterface.completedChunkTransferSize[expectedChunkUploadId], Int64(tempData.count) - preexistingChunk.size ) From 3cc3ac0b2e19c192363e3940b96fee2c30652e75 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 17:17:24 +0800 Subject: [PATCH 77/86] Remove period from upload logging Signed-off-by: Claudio Cambra --- Sources/NextcloudFileProviderKit/Utilities/Upload.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 495fa21c..19637104 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -68,7 +68,7 @@ func upload( """ Performing chunked upload to \(remotePath, privacy: .public) localFilePath: \(localFilePath, privacy: .public) - remoteChunkStoreFolderName: \(chunkUploadId, privacy: .public). + remoteChunkStoreFolderName: \(chunkUploadId, privacy: .public) chunkSize: \(chunkSize, privacy: .public) """ ) From c2b6876882518223c7751294ac2e16638d9ad395 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 18:55:08 +0800 Subject: [PATCH 78/86] Set custom file chunks directory when using nckit chunked upload Signed-off-by: Claudio Cambra --- .../Interface/NextcloudKit+RemoteInterface.swift | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 9f2e5c75..faf09a2f 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -110,9 +110,24 @@ extension NextcloudKit: RemoteInterface { } let localUrl = URL(fileURLWithPath: localPath) + let fm = FileManager.default + let chunksOutputDirectoryUrl = + fm.temporaryDirectory.appendingPathComponent(remoteChunkStoreFolderName) + do { + try fm.createDirectory(at: chunksOutputDirectoryUrl, withIntermediateDirectories: true) + } catch let error { + uploadLogger.error( + """ + Could not create temporary directory for chunked files: \(error, privacy: .public) + """ + ) + return ("", nil, nil, nil, .urlError) + } + return await withCheckedContinuation { continuation in uploadChunk( directory: localUrl.deletingLastPathComponent().path, + fileChunksOutputDirectory: chunksOutputDirectoryUrl.path, fileName: localUrl.lastPathComponent, destinationFileName: remoteUrl.lastPathComponent, date: modificationDate, From 3e580de879ce6b6deaad6ed122c98cb7eceecc76 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 8 Jan 2025 23:49:36 +0800 Subject: [PATCH 79/86] Add additional logging when starting nckit chunked upload Signed-off-by: Claudio Cambra --- .../NextcloudKit+RemoteInterface.swift | 35 +++++++++++++++---- .../Utilities/Upload.swift | 3 ++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index faf09a2f..bf415004 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -124,17 +124,40 @@ extension NextcloudKit: RemoteInterface { return ("", nil, nil, nil, .urlError) } + let directory = localUrl.deletingLastPathComponent().path + let fileChunksOutputDirectory = chunksOutputDirectoryUrl.path + let fileName = localUrl.lastPathComponent + let destinationFileName = remoteUrl.lastPathComponent + let serverUrl = remoteUrl.deletingLastPathComponent().absoluteString + let fileChunks = remainingChunks.toNcKitChunks() + + uploadLogger.info( + """ + Beginning chunked upload of: \(localPath, privacy: .public) + directory: \(directory, privacy: .public) + fileChunksOutputDirectory: \(fileChunksOutputDirectory, privacy: .public) + fileName: \(fileName, privacy: .public) + destinationFileName: \(destinationFileName, privacy: .public) + date: \(modificationDate?.debugDescription ?? "", privacy: .public) + creationDate: \(creationDate?.debugDescription ?? "", privacy: .public) + serverUrl: \(serverUrl, privacy: .public) + chunkFolder: \(remoteChunkStoreFolderName, privacy: .public) + filesChunk: \(fileChunks, privacy: .public) + chunkSize: \(chunkSize, privacy: .public) + """ + ) + return await withCheckedContinuation { continuation in uploadChunk( - directory: localUrl.deletingLastPathComponent().path, - fileChunksOutputDirectory: chunksOutputDirectoryUrl.path, - fileName: localUrl.lastPathComponent, - destinationFileName: remoteUrl.lastPathComponent, + directory: directory, + fileChunksOutputDirectory: fileChunksOutputDirectory, + fileName: fileName, + destinationFileName: destinationFileName, date: modificationDate, creationDate: creationDate, - serverUrl: remoteUrl.deletingLastPathComponent().absoluteString, + serverUrl: serverUrl, chunkFolder: remoteChunkStoreFolderName, - filesChunk: remainingChunks.toNcKitChunks(), + filesChunk: fileChunks, chunkSize: chunkSize, account: account.ncKitAccount, options: options, diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 19637104..25b611ad 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -115,6 +115,7 @@ func upload( taskHandler: taskHandler, progressHandler: progressHandler, chunkUploadCompleteHandler: { chunk in + uploadLogger.info("\(localFilePath, privacy: .public) uploaded chunk!") let db = dbManager.ncDatabase() do { try db.write { @@ -137,6 +138,8 @@ func upload( } ) + uploadLogger.info("\(localFilePath, privacy: .public) successfully uploaded in chunks") + return ( ocId: file?.ocId, chunks: chunks, From 73dd1c87cd91d9447fffa840a72e82de2b82bc1b Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Thu, 9 Jan 2025 01:09:15 +0800 Subject: [PATCH 80/86] Ensure chunk upload using nckit occurs on non-main queue Internally this method uses a dispatch semaphore to wait for each chunk upload to be complete; if this is done on the main queue the process locks up Signed-off-by: Claudio Cambra --- .../Interface/NextcloudKit+RemoteInterface.swift | 1 + Sources/NextcloudFileProviderKit/Item/Item+Create.swift | 2 -- Sources/NextcloudFileProviderKit/Item/Item+Modify.swift | 2 -- Sources/NextcloudFileProviderKit/Utilities/Upload.swift | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index bf415004..59fa1b62 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -172,6 +172,7 @@ extension NextcloudKit: RemoteInterface { requestHandler: requestHandler, taskHandler: taskHandler, progressHandler: { totalBytesExpected, totalBytes, fractionCompleted in + uploadLogger.info("Chunk progress: \(fractionCompleted * 100, privacy: .public)%") let currentProgress = Progress(totalUnitCount: totalBytesExpected) currentProgress.completedUnitCount = totalBytes progressHandler(currentProgress) diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift index 1b3133c0..b101812b 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Create.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Create.swift @@ -116,7 +116,6 @@ extension Item { dbManager: dbManager, creationDate: itemTemplate.creationDate as? Date, modificationDate: itemTemplate.contentModificationDate as? Date, - options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in if let domain = domain { @@ -315,7 +314,6 @@ extension Item { dbManager: dbManager, creationDate: childUrlAttributes.creationDate, modificationDate: childUrlAttributes.contentModificationDate, - options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in if let domain { diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift index c2038301..26f5958d 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift @@ -141,7 +141,6 @@ public extension Item { dbManager: dbManager, creationDate: newCreationDate, modificationDate: newContentModificationDate, - options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in if let domain { @@ -400,7 +399,6 @@ public extension Item { dbManager: dbManager, creationDate: childUrlAttributes.creationDate, modificationDate: childUrlAttributes.contentModificationDate, - options: .init(), requestHandler: { progress.setHandlersFromAfRequest($0) }, taskHandler: { task in if let domain { diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 25b611ad..945056e5 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -24,7 +24,7 @@ func upload( dbManager: FilesDatabaseManager = .shared, creationDate: Date? = nil, modificationDate: Date? = nil, - options: NKRequestOptions = .init(), + options: NKRequestOptions = .init(queue: .global(qos: .utility)), requestHandler: @escaping (UploadRequest) -> Void = { _ in }, taskHandler: @escaping (URLSessionTask) -> Void = { _ in }, progressHandler: @escaping (Progress) -> Void = { _ in }, From f2f2d3cde2e9f809132583dca7f926a69349b01f Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Thu, 9 Jan 2025 12:24:45 +0800 Subject: [PATCH 81/86] Fix issues with urls used in nckit extension for chunked uploads Signed-off-by: Claudio Cambra --- .../Interface/NextcloudKit+RemoteInterface.swift | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 59fa1b62..326b59f9 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -124,11 +124,23 @@ extension NextcloudKit: RemoteInterface { return ("", nil, nil, nil, .urlError) } - let directory = localUrl.deletingLastPathComponent().path + var directory = localUrl.deletingLastPathComponent().path + if directory.last == "/" { + directory.removeLast() + } let fileChunksOutputDirectory = chunksOutputDirectoryUrl.path let fileName = localUrl.lastPathComponent let destinationFileName = remoteUrl.lastPathComponent - let serverUrl = remoteUrl.deletingLastPathComponent().absoluteString + guard let serverUrl = remoteUrl + .deletingLastPathComponent() + .absoluteString + .removingPercentEncoding + else { + uploadLogger.error( + "NCKit ext: Could not get server url from \(remotePath, privacy: .public)" + ) + return ("", nil, nil, nil, .urlError) + } let fileChunks = remainingChunks.toNcKitChunks() uploadLogger.info( From 31cbb27f6f201488143712cc3307059831e2d5bd Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Thu, 9 Jan 2025 13:02:28 +0800 Subject: [PATCH 82/86] Remove over-active chunk progress logging Signed-off-by: Claudio Cambra --- .../Interface/NextcloudKit+RemoteInterface.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift index 326b59f9..428bee5e 100644 --- a/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift +++ b/Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift @@ -184,7 +184,6 @@ extension NextcloudKit: RemoteInterface { requestHandler: requestHandler, taskHandler: taskHandler, progressHandler: { totalBytesExpected, totalBytes, fractionCompleted in - uploadLogger.info("Chunk progress: \(fractionCompleted * 100, privacy: .public)%") let currentProgress = Progress(totalUnitCount: totalBytesExpected) currentProgress.completedUnitCount = totalBytes progressHandler(currentProgress) From c3bd5bf1b293e9e6fa0b556b85c7e5dbc866a8b7 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Thu, 9 Jan 2025 16:13:25 +0800 Subject: [PATCH 83/86] Simplify returns for upload function Signed-off-by: Claudio Cambra --- .../Utilities/Upload.swift | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index 945056e5..d722c80b 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -53,15 +53,7 @@ func upload( progressHandler: progressHandler ) - return ( - ocId: ocId, - chunks: nil, - etag: etag, - date: date as? Date, - size: size, - afError: afError, - remoteError: remoteError - ) + return (ocId, nil, etag, date as? Date, size, afError, remoteError) } uploadLogger.info( @@ -140,13 +132,5 @@ func upload( uploadLogger.info("\(localFilePath, privacy: .public) successfully uploaded in chunks") - return ( - ocId: file?.ocId, - chunks: chunks, - etag: file?.etag, - date: file?.date, - size: file?.size, - afError: afError, - remoteError: remoteError - ) + return (file?.ocId, chunks, file?.etag, file?.date, file?.size, afError, remoteError) } From a4b2e82d8d72b707f897f5c798858716d1dfc685 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Thu, 9 Jan 2025 16:54:09 +0800 Subject: [PATCH 84/86] Use setStatusForItemMetadata when setting post-upload metadata state in Item content modification Signed-off-by: Claudio Cambra --- .../Item/Item+Modify.swift | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift index 26f5958d..104b9ef9 100644 --- a/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift +++ b/Sources/NextcloudFileProviderKit/Item/Item+Modify.swift @@ -118,9 +118,9 @@ public extension Item { } let updatedMetadata = await withCheckedContinuation { continuation in - dbManager.setStatusForItemMetadata( - metadata, status: ItemMetadata.Status.uploading - ) { continuation.resume(returning: $0) } + dbManager.setStatusForItemMetadata(metadata, status: .uploading) { + continuation.resume(returning: $0) + } } if updatedMetadata == nil { @@ -188,7 +188,15 @@ public extension Item { ) } - let newMetadata = ItemMetadata(value: metadata) + let newMetadata: ItemMetadata = await { + guard let updatedMetadata else { return ItemMetadata(value: metadata) } + return await withCheckedContinuation { continuation in + dbManager.setStatusForItemMetadata(updatedMetadata, status: .normal) { updatedMeta in + continuation.resume(returning: updatedMeta ?? ItemMetadata(value: updatedMetadata)) + } + } + }() + newMetadata.date = date ?? Date() newMetadata.etag = etag ?? metadata.etag newMetadata.ocId = ocId @@ -196,10 +204,8 @@ public extension Item { newMetadata.session = "" newMetadata.sessionError = "" newMetadata.sessionTaskIdentifier = 0 - newMetadata.status = ItemMetadata.Status.normal.rawValue newMetadata.downloaded = true newMetadata.uploaded = true - newMetadata.chunkUploadId = "" dbManager.addItemMetadata(newMetadata) From 65d913a7bf129b8e71c25e2d71c3ef6a3583e261 Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Thu, 9 Jan 2025 17:28:38 +0800 Subject: [PATCH 85/86] Do not add chunks to database if we have chunkes logged in db Signed-off-by: Claudio Cambra --- .../NextcloudFileProviderKit/Utilities/Upload.swift | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift index d722c80b..3fb74501 100644 --- a/Sources/NextcloudFileProviderKit/Utilities/Upload.swift +++ b/Sources/NextcloudFileProviderKit/Utilities/Upload.swift @@ -90,7 +90,11 @@ func upload( ) }, chunkUploadStartHandler: { chunks in - uploadLogger.info("\(localFilePath, privacy: .public) uploading chunk") + uploadLogger.info("\(localFilePath, privacy: .public) chunked upload starting...") + + // Do not add chunks to database if we have done this already + guard remainingChunks.isEmpty else { return } + let db = dbManager.ncDatabase() do { try db.write { db.add(chunks.map { RemoteFileChunk(value: $0) }) } @@ -107,7 +111,9 @@ func upload( taskHandler: taskHandler, progressHandler: progressHandler, chunkUploadCompleteHandler: { chunk in - uploadLogger.info("\(localFilePath, privacy: .public) uploaded chunk!") + uploadLogger.info( + "\(localFilePath, privacy: .public) chunk \(chunk.fileName, privacy: .public) done" + ) let db = dbManager.ncDatabase() do { try db.write { From 07b82add520f12f24dfab599fcc9c81d76da53da Mon Sep 17 00:00:00 2001 From: Claudio Cambra Date: Wed, 15 Jan 2025 11:10:24 +0800 Subject: [PATCH 86/86] Switch back to develop NextcloudKit branch Signed-off-by: Claudio Cambra --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 288d5c88..7926455d 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( url: "https://github.com/claucambra/NextcloudCapabilitiesKit.git", .upToNextMajor(from: "2.0.0") ), - .package(url: "https://github.com/nextcloud/NextcloudKit", branch: "work/chunk-destination-filename"), + .package(url: "https://github.com/nextcloud/NextcloudKit", branch: "develop"), .package(url: "https://github.com/realm/realm-swift.git", exact: "10.49.0"), .package(url: "https://github.com/apple/swift-nio.git", from: "2.0.0") ],