Skip to content

chunk #167

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
marinofaggiana merged 6 commits into from
Jul 17, 2025
Merged

chunk #167

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
64a712b
improved
marinofaggiana Jul 17, 2025
f975651
fix eof
marinofaggiana Jul 17, 2025
c7504fc
cod
marinofaggiana Jul 17, 2025
35b4201
fix error
marinofaggiana Jul 17, 2025
dfc7369
added parameter
marinofaggiana Jul 17, 2025
c376da1
x3
marinofaggiana Jul 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 85 additions & 42 deletions Sources/NextcloudKit/NKCommon.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,27 +91,27 @@ public struct NKCommon: Sendable {
filesChunk: [(fileName: String, size: Int64)],
numChunks: @escaping (_ num: Int) -> Void = { _ in },
counterChunk: @escaping (_ counter: Int) -> Void = { _ in },
completion: @escaping (_ filesChunk: [(fileName: String, size: Int64)]) -> Void = { _ in }) {
// Check if filesChunk is empty
completion: @escaping (_ filesChunk: [(fileName: String, size: Int64)], _ error: Error?) -> Void = { _, _ in }) {
// Return existing chunks immediately
if !filesChunk.isEmpty {
return completion(filesChunk)
return completion(filesChunk, nil)
}

defer {
NotificationCenter.default.removeObserver(self, name: notificationCenterChunkedFileStop, object: nil)
}

let fileManager: FileManager = .default
let fileManager = FileManager.default
var isDirectory: ObjCBool = false
var reader: FileHandle?
var writer: FileHandle?
var chunk: Int = 0
var counter: Int = 1
var chunkWrittenBytes = 0
var counter = 1
var incrementalSize: Int64 = 0
var filesChunk: [(fileName: String, size: Int64)] = []
var chunkSize = chunkSize
let bufferSize = 1000000
var stop: Bool = false
let bufferSize = 1_000_000
var stop = false

NotificationCenter.default.addObserver(forName: notificationCenterChunkedFileStop, object: nil, queue: nil) { _ in
stop = true
Expand All @@ -122,78 +122,121 @@ public struct NKCommon: Sendable {
let totalSize = getFileSize(filePath: inputFilePath)
var num: Int = Int(totalSize / Int64(chunkSize))

if num > 10000 {
if num > 10_000 {
chunkSize += 100_000_000
num = Int(totalSize / Int64(chunkSize)) // ricalcolo
num = Int(totalSize / Int64(chunkSize))
}
numChunks(num)

// Create output directory if needed
if !fileManager.fileExists(atPath: outputDirectory, isDirectory: &isDirectory) {
do {
try fileManager.createDirectory(atPath: outputDirectory, withIntermediateDirectories: true, attributes: nil)
} catch {
return completion([])
return completion([], NSError(domain: "chunkedFile", code: -2,userInfo: [NSLocalizedDescriptionKey: "Failed to create the output directory for file chunks."]))
}
}

// Open input file
do {
reader = try .init(forReadingFrom: URL(fileURLWithPath: inputFilePath))
} catch {
return completion([])
return completion([], NSError(domain: "chunkedFile", code: -1, userInfo: [NSLocalizedDescriptionKey: "Failed to open the input file for reading."]))
}

repeat {
outerLoop: repeat {
if stop {
return completion([])
return completion([], NSError(domain: "chunkedFile", code: -5, userInfo: [NSLocalizedDescriptionKey: "Chunking was stopped by user request or system notification."]))
}
if autoreleasepool(invoking: { () -> Int in
if chunk >= chunkSize {
writer?.closeFile()
writer = nil
chunk = 0
counterChunk(counter)
debugPrint("[DEBUG] Counter: \(counter)")
counter += 1

let result = autoreleasepool(invoking: { () -> Int in
let remaining = chunkSize - chunkWrittenBytes
guard let rawBuffer = reader?.readData(ofLength: min(bufferSize, remaining)) else {
return -1 // Error: read failed
}

let chunkRemaining: Int = chunkSize - chunk
let rawBuffer = reader?.readData(ofLength: min(bufferSize, chunkRemaining))
if rawBuffer.isEmpty {
// Final flush of last chunk
if writer != nil {
writer?.closeFile()
writer = nil
counterChunk(counter)
debugPrint("[DEBUG] Final chunk closed: \(counter)")
counter += 1
}
return 0 // End of file
}

let safeBuffer = Data(rawBuffer)


if writer == nil {
let fileNameChunk = String(counter)
let outputFileName = outputDirectory + "/" + fileNameChunk
fileManager.createFile(atPath: outputFileName, contents: nil, attributes: nil)
do {
writer = try .init(forWritingTo: URL(fileURLWithPath: outputFileName))
writer = try FileHandle(forWritingTo: URL(fileURLWithPath: outputFileName))
} catch {
filesChunk = []
return 0
return -2 // Error: cannot create writer
}
filesChunk.append((fileName: fileNameChunk, size: 0))
}

if let rawBuffer = rawBuffer {
let safeBuffer = Data(rawBuffer) // secure copy
writer?.write(safeBuffer)
chunk = chunk + safeBuffer.count
return safeBuffer.count
// Check free disk space
if let free = try? URL(fileURLWithPath: outputDirectory)
.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey])
.volumeAvailableCapacityForImportantUsage,
free < Int64(safeBuffer.count * 2) {
return -3 // Not enough disk space
}
filesChunk = []
return 0
}) == 0 { break }

do {
try writer?.write(contentsOf: safeBuffer)
chunkWrittenBytes += safeBuffer.count
if chunkWrittenBytes >= chunkSize {
writer?.closeFile()
writer = nil
chunkWrittenBytes = 0
counterChunk(counter)
debugPrint("[DEBUG] Chunk completed: \(counter)")
counter += 1
}
return 1 // OK
} catch {
return -4 // Write error
}
})

switch result {
case -1:
return completion([], NSError(domain: "chunkedFile", code: -1, userInfo: [NSLocalizedDescriptionKey: "Failed to read data from the input file."]))
case -2:
return completion([], NSError(domain: "chunkedFile", code: -2, userInfo: [NSLocalizedDescriptionKey: "Failed to open the output chunk file for writing."]))
case -3:
return completion([], NSError(domain: "chunkedFile", code: -3, userInfo: [NSLocalizedDescriptionKey: "There is not enough available disk space to proceed."]))
case -4:
return completion([], NSError(domain: "chunkedFile", code: -4, userInfo: [NSLocalizedDescriptionKey: "Failed to write data to chunk file."]))
case 0:
break outerLoop
case 1:
continue
default:
break
}
} while true

writer?.closeFile()
reader?.closeFile()

counter = 0
for fileChunk in filesChunk {
let size = getFileSize(filePath: outputDirectory + "/" + fileChunk.fileName)
incrementalSize = incrementalSize + size
filesChunk[counter].size = incrementalSize
counter += 1
// Update incremental chunk sizes
for i in 0..<filesChunk.count {
let path = outputDirectory + "/" + filesChunk[i].fileName
let size = getFileSize(filePath: path)
incrementalSize += size
filesChunk[i].size = incrementalSize
}
return completion(filesChunk)

completion(filesChunk, nil)
}

// MARK: - Server Error GroupDefaults
Expand Down
57 changes: 33 additions & 24 deletions Sources/NextcloudKit/NextcloudKit+Upload.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ public extension NextcloudKit {
taskHandler: @escaping (_ task: URLSessionTask) -> Void = { _ in },
progressHandler: @escaping (_ totalBytesExpected: Int64, _ totalBytes: Int64, _ fractionCompleted: Double) -> Void = { _, _, _ in },
uploaded: @escaping (_ fileChunk: (fileName: String, size: Int64)) -> Void = { _ in },
assemble: @escaping () -> Void = { },
completion: @escaping (_ account: String, _ filesChunk: [(fileName: String, size: Int64)]?, _ file: NKFile?, _ error: NKError) -> Void) {
guard let nkSession = nkCommonInstance.nksessions.session(forAccount: account) else {
return completion(account, nil, nil, .urlError)
Expand All @@ -233,7 +234,7 @@ public extension NextcloudKit {
options.customHeader?["Destination"] = serverUrlFileName.urlEncoded
options.customHeader?["OC-Total-Length"] = String(fileNameLocalSize)

// check space
// Check available disk space
#if os(macOS)
var fsAttributes: [FileAttributeKey: Any]
do {
Expand Down Expand Up @@ -266,13 +267,14 @@ public extension NextcloudKit {
#endif

#if os(visionOS) || os(iOS)
if freeDisk < fileNameLocalSize * 2 {
if freeDisk < fileNameLocalSize * 3 {
// It seems there is not enough space to send the file
return completion(account, nil, nil, .errorChunkNoEnoughMemory)
}
#endif

func createFolder(completion: @escaping (_ errorCode: NKError) -> Void) {
// Ensure upload chunk folder exists
func createFolderIfNeeded(completion: @escaping (_ errorCode: NKError) -> Void) {
readFileOrFolder(serverUrlFileName: serverUrlChunkFolder, depth: "0", account: account, options: options) { _, _, _, error in
if error == .success {
completion(NKError())
Expand All @@ -286,33 +288,46 @@ public extension NextcloudKit {
}
}

createFolder { error in
createFolderIfNeeded { error in
guard error == .success else {
return completion(account, nil, nil, .errorChunkCreateFolder)
}
let outputDirectory = fileChunksOutputDirectory ?? directory
var uploadNKError = NKError()

let outputDirectory = fileChunksOutputDirectory ?? directory
self.nkCommonInstance.chunkedFile(inputDirectory: directory, outputDirectory: outputDirectory, fileName: fileName, chunkSize: chunkSize, filesChunk: filesChunk) { num in

self.nkCommonInstance.chunkedFile(inputDirectory: directory,
outputDirectory: outputDirectory,
fileName: fileName,
chunkSize: chunkSize,
filesChunk: filesChunk) { num in
numChunks(num)
} counterChunk: { counter in
counterChunk(counter)
} completion: { filesChunk in
if filesChunk.isEmpty {
// The file for sending could not be created
return completion(account, nil, nil, .errorChunkFilesEmpty)
} completion: { filesChunk, error in

// Check chunking error
if let error {
return completion(account, nil, nil, NKError(error: error))
}

guard !filesChunk.isEmpty else {
return completion(account, nil, nil, NKError(error: NSError(domain: "chunkedFile", code: -5,userInfo: [NSLocalizedDescriptionKey: "Files empty."])))
}

var filesChunkOutput = filesChunk
start(filesChunkOutput)

for fileChunk in filesChunk {
let serverUrlFileName = serverUrlChunkFolder + "/" + fileChunk.fileName
let fileNameLocalPath = outputDirectory + "/" + fileChunk.fileName
let fileSize = self.nkCommonInstance.getFileSize(filePath: fileNameLocalPath)

if fileSize == 0 {
// The file could not be sent
return completion(account, nil, nil, .errorChunkFileNull)
return completion(account, nil, nil, NKError(error: NSError(domain: "chunkedFile", code: -6,userInfo: [NSLocalizedDescriptionKey: "File empty."])))
}

let semaphore = DispatchSemaphore(value: 0)
self.upload(serverUrlFileName: serverUrlFileName, fileNameLocalPath: fileNameLocalPath, account: account, options: options, requestHandler: { request in
requestHandler(request)
Expand Down Expand Up @@ -359,6 +374,8 @@ public extension NextcloudKit {
let assembleTimeMax: Double = 30 * 60 // 30 min
options.timeout = max(assembleTimeMin, min(assembleTimePerGB * assembleSizeInGB, assembleTimeMax))

assemble()

self.moveFileOrFolder(serverUrlFileNameSource: serverUrlFileNameSource, serverUrlFileNameDestination: serverUrlFileName, overwrite: true, account: account, options: options) { _, _, error in
guard error == .success else {
return completion(account, filesChunkOutput, nil,.errorChunkMoveFile)
Expand Down Expand Up @@ -400,13 +417,9 @@ public extension NextcloudKit {
requestHandler: @escaping (_ request: UploadRequest) -> Void = { _ in },
taskHandler: @escaping (_ task: URLSessionTask) -> Void = { _ in },
progressHandler: @escaping (_ totalBytesExpected: Int64, _ totalBytes: Int64, _ fractionCompleted: Double) -> Void = { _, _, _ in },
assemble: @escaping () -> Void = { },
uploaded: @escaping (_ fileChunk: (fileName: String, size: Int64)) -> Void = { _ in }
) async -> (
account: String,
remainingChunks: [(fileName: String, size: Int64)]?,
file: NKFile?,
error: NKError
) {
) async -> (account: String, remainingChunks: [(fileName: String, size: Int64)]?, file: NKFile?, error: NKError) {
await withCheckedContinuation { continuation in
uploadChunk(directory: directory,
fileChunksOutputDirectory: fileChunksOutputDirectory,
Expand All @@ -426,13 +439,9 @@ public extension NextcloudKit {
requestHandler: requestHandler,
taskHandler: taskHandler,
progressHandler: progressHandler,
uploaded: uploaded) { account, remaining, file, error in
continuation.resume(returning: (
account: account,
remainingChunks: remaining,
file: file,
error: error
))
uploaded: uploaded,
assemble: assemble) { account, remaining, file, error in
continuation.resume(returning: (account: account, remainingChunks: remaining, file: file, error: error))
}
}
}
Expand Down
Loading