Implement chunked uploads

Sources/NextcloudFileProviderKit/Database/FilesDatabaseManager.swift

@@ -89,7 +89,7 @@ public class FilesDatabaseManager {
                 }
 
             },
-            objectTypes: [ItemMetadata.self]
+            objectTypes: [ItemMetadata.self, RemoteFileChunk.self]
         )
         self.init(realmConfig: config)
     }
@@ -340,6 +340,9 @@ public class FilesDatabaseManager {
                     result.downloaded = false
                 } else if result.isUpload {
                     result.uploaded = false
+                    result.chunkUploadId = UUID().uuidString
+                } else if status == .normal, metadata.isUpload {
+                    result.chunkUploadId = ""
                 }
 
                 Self.logger.debug(

Sources/NextcloudFileProviderKit/Extensions/Results+Extensions.swift

@@ -8,8 +8,17 @@
 import Realm
 import RealmSwift
 
+extension Results where Element: RemoteFileChunk {
+    func toUnmanagedResults() -> [RemoteFileChunk] {
+        return map { RemoteFileChunk(value: $0) }
+    }
+}
+
 extension Results where Element: ItemMetadata {
     func toUnmanagedResults() -> [ItemMetadata] {
         return map { ItemMetadata(value: $0) }
     }
 }
+
+
+

Sources/NextcloudFileProviderKit/Interface/NextcloudKit+RemoteInterface.swift

@@ -80,6 +80,130 @@ extension NextcloudKit: RemoteInterface {
         }
     }
 
+    public func chunkedUpload(
+        localPath: String,
+        remotePath: String,
+        remoteChunkStoreFolderName: String = UUID().uuidString,
+        chunkSize: Int,
+        remainingChunks: [RemoteFileChunk],
+        creationDate: Date? = nil,
+        modificationDate: Date? = nil,
+        account: Account,
+        options: NKRequestOptions = .init(),
+        currentNumChunksUpdateHandler: @escaping (_ num: Int) -> Void = { _ in },
+        chunkCounter: @escaping (_ counter: Int) -> Void = { _ in },
+        chunkUploadStartHandler: @escaping (_ filesChunk: [RemoteFileChunk]) -> Void = { _ in },
+        requestHandler: @escaping (_ request: UploadRequest) -> Void = { _ in },
+        taskHandler: @escaping (_ task: URLSessionTask) -> Void = { _ in },
+        progressHandler: @escaping (Progress) -> Void = { _ in },
+        chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void = { _ in }
+    ) async -> (
+        account: String,
+        fileChunks: [RemoteFileChunk]?,
+        file: NKFile?,
+        afError: AFError?,
+        remoteError: NKError
+    ) {
+        guard let remoteUrl = URL(string: remotePath) else {
+            uploadLogger.error("NCKit ext: Could not get url from \(remotePath, privacy: .public)")
+            return ("", nil, nil, nil, .urlError)
+        }
+        let localUrl = URL(fileURLWithPath: localPath)
+
+        let fm = FileManager.default
+        let chunksOutputDirectoryUrl =
+            fm.temporaryDirectory.appendingPathComponent(remoteChunkStoreFolderName)
+        do {
+            try fm.createDirectory(at: chunksOutputDirectoryUrl, withIntermediateDirectories: true)
+        } catch let error {
+            uploadLogger.error(
+                """
+                Could not create temporary directory for chunked files: \(error, privacy: .public)
+                """
+            )
+            return ("", nil, nil, nil, .urlError)
+        }
+
+        var directory = localUrl.deletingLastPathComponent().path
+        if directory.last == "/" {
+            directory.removeLast()
+        }
+        let fileChunksOutputDirectory = chunksOutputDirectoryUrl.path
+        let fileName = localUrl.lastPathComponent
+        let destinationFileName = remoteUrl.lastPathComponent
+        guard let serverUrl = remoteUrl
+            .deletingLastPathComponent()
+            .absoluteString
+            .removingPercentEncoding
+        else {
+            uploadLogger.error(
+                "NCKit ext: Could not get server url from \(remotePath, privacy: .public)"
+            )
+            return ("", nil, nil, nil, .urlError)
+        }
+        let fileChunks = remainingChunks.toNcKitChunks()
+
+        uploadLogger.info(
+            """
+            Beginning chunked upload of: \(localPath, privacy: .public)
+                directory: \(directory, privacy: .public)
+                fileChunksOutputDirectory: \(fileChunksOutputDirectory, privacy: .public)
+                fileName: \(fileName, privacy: .public)
+                destinationFileName: \(destinationFileName, privacy: .public)
+                date: \(modificationDate?.debugDescription ?? "", privacy: .public)
+                creationDate: \(creationDate?.debugDescription ?? "", privacy: .public)
+                serverUrl: \(serverUrl, privacy: .public)
+                chunkFolder: \(remoteChunkStoreFolderName, privacy: .public)
+                filesChunk: \(fileChunks, privacy: .public)
+                chunkSize: \(chunkSize, privacy: .public)
+            """
+        )
+
+        return await withCheckedContinuation { continuation in
+            uploadChunk(
+                directory: directory,
+                fileChunksOutputDirectory: fileChunksOutputDirectory,
+                fileName: fileName,
+                destinationFileName: destinationFileName,
+                date: modificationDate,
+                creationDate: creationDate,
+                serverUrl: serverUrl,
+                chunkFolder: remoteChunkStoreFolderName,
+                filesChunk: fileChunks,
+                chunkSize: chunkSize,
+                account: account.ncKitAccount,
+                options: options,
+                numChunks: currentNumChunksUpdateHandler,
+                counterChunk: chunkCounter,
+                start: { processedChunks in
+                    let chunks = RemoteFileChunk.fromNcKitChunks(
+                        processedChunks, remoteChunkStoreFolderName: remoteChunkStoreFolderName
+                    )
+                    chunkUploadStartHandler(chunks)
+                },
+                requestHandler: requestHandler,
+                taskHandler: taskHandler,
+                progressHandler: { totalBytesExpected, totalBytes, fractionCompleted in
+                    let currentProgress = Progress(totalUnitCount: totalBytesExpected)
+                    currentProgress.completedUnitCount = totalBytes
+                    progressHandler(currentProgress)
+                },
+                uploaded: { uploadedChunk in
+                    let chunk = RemoteFileChunk(
+                        ncKitChunk: uploadedChunk,
+                        remoteChunkStoreFolderName: remoteChunkStoreFolderName
+                    )
+                    chunkUploadCompleteHandler(chunk)
+                }
+            ) { account, receivedChunks, file, afError, error in
+                let chunks = RemoteFileChunk.fromNcKitChunks(
+                    receivedChunks ?? [], remoteChunkStoreFolderName: remoteChunkStoreFolderName
+                )
+                continuation.resume(returning: (account, chunks, file, afError, error))
+            }
+        }
+    }
+
     public func move(
         remotePathSource: String,
         remotePathDestination: String,

Sources/NextcloudFileProviderKit/Interface/RemoteInterface.swift

@@ -52,6 +52,31 @@ public protocol RemoteInterface {
         remoteError: NKError
     )
 
+    func chunkedUpload(
+        localPath: String,
+        remotePath: String,
+        remoteChunkStoreFolderName: String,
+        chunkSize: Int,
+        remainingChunks: [RemoteFileChunk],
+        creationDate: Date?,
+        modificationDate: Date?,
+        account: Account,
+        options: NKRequestOptions,
+        currentNumChunksUpdateHandler: @escaping (_ num: Int) -> Void,
+        chunkCounter: @escaping (_ counter: Int) -> Void,
+        chunkUploadStartHandler: @escaping (_ filesChunk: [RemoteFileChunk]) -> Void,
+        requestHandler: @escaping (_ request: UploadRequest) -> Void,
+        taskHandler: @escaping (_ task: URLSessionTask) -> Void,
+        progressHandler: @escaping (Progress) -> Void,
+        chunkUploadCompleteHandler: @escaping (_ fileChunk: RemoteFileChunk) -> Void
+    ) async -> (
+        account: String,
+        fileChunks: [RemoteFileChunk]?,
+        file: NKFile?,
+        afError: AFError?,
+        remoteError: NKError
+    )
+
     func move(
         remotePathSource: String,
         remotePathDestination: String,

Sources/NextcloudFileProviderKit/Item/Item+Create.swift

@@ -105,13 +105,17 @@ extension Item {
         progress: Progress,
         dbManager: FilesDatabaseManager
     ) async -> (Item?, Error?) {
-        let (_, ocId, etag, date, size, _, _, error) = await remoteInterface.upload(
-            remotePath: remotePath,
-            localPath: localPath,
+        let chunkUploadId =
+            itemTemplate.itemIdentifier.rawValue.replacingOccurrences(of: "/", with: "")
+        let (ocId, _, etag, date, size, _, error) = await upload(
+            fileLocatedAt: localPath,
+            toRemotePath: remotePath,
+            usingRemoteInterface: remoteInterface,
+            withAccount: account,
+            usingChunkUploadId: chunkUploadId,
+            dbManager: dbManager,
             creationDate: itemTemplate.creationDate as? Date,
             modificationDate: itemTemplate.contentModificationDate as? Date,
-            account: account,
-            options: .init(),
             requestHandler: { progress.setHandlersFromAfRequest($0) },
             taskHandler: { task in
                 if let domain = domain {
@@ -147,8 +151,8 @@ extension Item {
             filename: \(itemTemplate.filename, privacy: .public)
             ocId: \(ocId, privacy: .public)
             etag: \(etag ?? "", privacy: .public)
-            date: \(date ?? NSDate(), privacy: .public)
-            size: \(size, privacy: .public),
+            date: \(date ?? Date(), privacy: .public)
+            size: \(Int(size ?? -1), privacy: .public),
             account: \(account.ncKitAccount, privacy: .public)
             """
         )
@@ -157,20 +161,20 @@ extension Item {
             Self.logger.warning(
                 """
                 Created item upload reported as successful, but there are differences between
-                the received file size (\(size, privacy: .public))
+                the received file size (\(Int(size ?? -1), privacy: .public))
                 and the original file size (\(itemTemplate.documentSize??.int64Value ?? 0))
                 """
             )
         }
 
         let newMetadata = ItemMetadata()
-        newMetadata.date = (date ?? NSDate()) as Date
+        newMetadata.date = date ?? Date()
         newMetadata.etag = etag ?? ""
         newMetadata.account = account.ncKitAccount
         newMetadata.fileName = itemTemplate.filename
         newMetadata.fileNameView = itemTemplate.filename
         newMetadata.ocId = ocId
-        newMetadata.size = size
+        newMetadata.size = size ?? 0
         newMetadata.contentType = itemTemplate.contentType?.preferredMIMEType ?? ""
         newMetadata.directory = false
         newMetadata.serverUrl = parentItemRemotePath
@@ -302,13 +306,14 @@ extension Item {
                     Handling child bundle or package file at: \(childUrlPath, privacy: .public)
                     """
                 )
-                let (_, _, _, _, _, _, _, error) = await remoteInterface.upload(
-                    remotePath: childRemoteUrl,
-                    localPath: childUrlPath,
+                let (_, _, _, _, _, _, error) = await upload(
+                    fileLocatedAt: childUrlPath,
+                    toRemotePath: childRemoteUrl,
+                    usingRemoteInterface: remoteInterface,
+                    withAccount: account,
+                    dbManager: dbManager,
                     creationDate: childUrlAttributes.creationDate,
                     modificationDate: childUrlAttributes.contentModificationDate,
-                    account: account,
-                    options: .init(),
                     requestHandler: { progress.setHandlersFromAfRequest($0) },
                     taskHandler: { task in
                         if let domain {

Sources/NextcloudFileProviderKit/Item/Item+Modify.swift

@@ -118,9 +118,9 @@ public extension Item {
         }
 
         let updatedMetadata = await withCheckedContinuation { continuation in
-            dbManager.setStatusForItemMetadata(
-                metadata, status: ItemMetadata.Status.uploading
-            ) { continuation.resume(returning: $0) }
+            dbManager.setStatusForItemMetadata(metadata, status: .uploading) {
+                continuation.resume(returning: $0)
+            }
         }
 
         if updatedMetadata == nil {
@@ -132,13 +132,15 @@ public extension Item {
             )
         }
 
-        let (_, _, etag, date, size, _, _, error) = await remoteInterface.upload(
-            remotePath: remotePath,
-            localPath: newContents.path,
+        let (_, _, etag, date, size, _, error) = await upload(
+            fileLocatedAt: newContents.path,
+            toRemotePath: remotePath,
+            usingRemoteInterface: remoteInterface,
+            withAccount: account,
+            usingChunkUploadId: metadata.chunkUploadId,
+            dbManager: dbManager,
             creationDate: newCreationDate,
             modificationDate: newContentModificationDate,
-            account: account,
-            options: .init(),
             requestHandler: { progress.setHandlersFromAfRequest($0) },
             taskHandler: { task in
                 if let domain {
@@ -180,21 +182,28 @@ public extension Item {
             Self.logger.warning(
                 """
                 Item content modification upload reported as successful,
-                but there are differences between the received file size (\(size, privacy: .public))
+                but there are differences between the received file size (\(size ?? -1, privacy: .public))
                 and the original file size (\(self.documentSize?.int64Value ?? 0))
                 """
             )
         }
 
-        let newMetadata = ItemMetadata(value: metadata)
-        newMetadata.date = (date ?? NSDate()) as Date
+        let newMetadata: ItemMetadata = await {
+            guard let updatedMetadata else { return ItemMetadata(value: metadata) }
+            return await withCheckedContinuation { continuation in
+                dbManager.setStatusForItemMetadata(updatedMetadata, status: .normal) { updatedMeta in
+                    continuation.resume(returning: updatedMeta ?? ItemMetadata(value: updatedMetadata))
+                }
+            }
+        }()
+
+        newMetadata.date = date ?? Date()
         newMetadata.etag = etag ?? metadata.etag
         newMetadata.ocId = ocId
-        newMetadata.size = size
+        newMetadata.size = size ?? 0
         newMetadata.session = ""
         newMetadata.sessionError = ""
         newMetadata.sessionTaskIdentifier = 0
-        newMetadata.status = ItemMetadata.Status.normal.rawValue
         newMetadata.downloaded = true
         newMetadata.uploaded = true
 
@@ -388,13 +397,14 @@ public extension Item {
                     Handling child bundle or package file at: \(childUrlPath, privacy: .public)
                     """
                 )
-                let (_, _, _, _, _, _, _, error) = await remoteInterface.upload(
-                    remotePath: childRemoteUrl,
-                    localPath: childUrlPath,
+                let (_, _, _, _, _, _, error) = await upload(
+                    fileLocatedAt: childUrlPath,
+                    toRemotePath: childRemoteUrl,
+                    usingRemoteInterface: remoteInterface,
+                    withAccount: account,
+                    dbManager: dbManager,
                     creationDate: childUrlAttributes.creationDate,
                     modificationDate: childUrlAttributes.contentModificationDate,
-                    account: account,
-                    options: .init(),
                     requestHandler: { progress.setHandlersFromAfRequest($0) },
                     taskHandler: { task in
                         if let domain {

Sources/NextcloudFileProviderKit/Metadata/ItemMetadata.swift

@@ -44,7 +44,7 @@ public class ItemMetadata: Object {
     @Persisted public var account = ""
     @Persisted public var assetLocalIdentifier = ""
     @Persisted public var checksums = ""
-    @Persisted public var chunk: Bool = false
+    @Persisted public var chunkUploadId: String = ""
     @Persisted public var classFile = ""
     @Persisted public var commentsUnread: Bool = false
     @Persisted public var contentType = ""