merge: sync upstream rclone/rclone changes with merged backends

CONTRIBUTING.md

@@ -628,7 +628,7 @@ You'll need to modify the following files
 - `backend/s3/s3.go`
   - Add the provider to `providerOption` at the top of the file
   - Add endpoints and other config for your provider gated on the provider in `fs.RegInfo`.
-  - Exclude your provider from generic config questions (eg `region` and `endpoint).
+  - Exclude your provider from generic config questions (eg `region` and `endpoint`).
   - Add the provider to the `setQuirks` function - see the documentation there.
 - `docs/content/s3.md`
   - Add the provider at the top of the page.

VERSION

@@ -1 +1 @@
-v1.73.1
+v1.73.2

backend/alias/alias.go

@@ -1,4 +1,4 @@
-// Package alias implements a virtual provider to rename existing remotes.
+// Package alias implements a virtual provider to rename existing remotes.
 package alias
 
 import (

backend/alias/alias_internal_test.go

@@ -1,4 +1,4 @@
-package alias
+package alias
 
 import (
 	"context"

backend/alist/alist.go

@@ -1,4 +1,4 @@
-// Package alist implements an rclone backend for AList
+// Package alist implements an rclone backend for AList
 package alist
 
 import (

backend/all/all.go

@@ -2,67 +2,71 @@
 package all
 
 import (
-	// Active file systems
-	_ "github.com/rclone/rclone/backend/alias"
-	_ "github.com/rclone/rclone/backend/alist"
-	_ "github.com/rclone/rclone/backend/alldebrid"
-	_ "github.com/rclone/rclone/backend/azureblob"
-	_ "github.com/rclone/rclone/backend/azurefiles"
-	_ "github.com/rclone/rclone/backend/b2"
-	_ "github.com/rclone/rclone/backend/box"
-	_ "github.com/rclone/rclone/backend/cache"
-	_ "github.com/rclone/rclone/backend/chunker"
-	_ "github.com/rclone/rclone/backend/cloudinary"
-	_ "github.com/rclone/rclone/backend/combine"
-	_ "github.com/rclone/rclone/backend/compress"
-	_ "github.com/rclone/rclone/backend/crypt"
-	_ "github.com/rclone/rclone/backend/drive"
-	_ "github.com/rclone/rclone/backend/dropbox"
-	_ "github.com/rclone/rclone/backend/fichier"
-	_ "github.com/rclone/rclone/backend/filefabric"
-	_ "github.com/rclone/rclone/backend/ftp"
-	_ "github.com/rclone/rclone/backend/googlecloudstorage"
-	_ "github.com/rclone/rclone/backend/googlephotos"
-	_ "github.com/rclone/rclone/backend/hasher"
-	_ "github.com/rclone/rclone/backend/hdfs"
-	_ "github.com/rclone/rclone/backend/hidrive"
-	_ "github.com/rclone/rclone/backend/http"
-	_ "github.com/rclone/rclone/backend/imagekit"
-	_ "github.com/rclone/rclone/backend/iclouddrive"
-	_ "github.com/rclone/rclone/backend/internetarchive"
-	_ "github.com/rclone/rclone/backend/jottacloud"
-	_ "github.com/rclone/rclone/backend/koofr"
-	_ "github.com/rclone/rclone/backend/linkbox"
-	_ "github.com/rclone/rclone/backend/local"
-	_ "github.com/rclone/rclone/backend/mailru"
-	_ "github.com/rclone/rclone/backend/mega"
-	_ "github.com/rclone/rclone/backend/memory"
-	_ "github.com/rclone/rclone/backend/netstorage"
-	_ "github.com/rclone/rclone/backend/onedrive"
-	_ "github.com/rclone/rclone/backend/opendrive"
-	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
-	_ "github.com/rclone/rclone/backend/pcloud"
-	_ "github.com/rclone/rclone/backend/pikpak"
-	_ "github.com/rclone/rclone/backend/premiumizeme"
-	_ "github.com/rclone/rclone/backend/protondrive"
-	_ "github.com/rclone/rclone/backend/putio"
-	_ "github.com/rclone/rclone/backend/qingstor"
-	_ "github.com/rclone/rclone/backend/quatrix"
-	_ "github.com/rclone/rclone/backend/s3"
-	_ "github.com/rclone/rclone/backend/seafile"
-	_ "github.com/rclone/rclone/backend/sftp"
-	_ "github.com/rclone/rclone/backend/sharefile"
-	_ "github.com/rclone/rclone/backend/sia"
-	_ "github.com/rclone/rclone/backend/smb"
-	_ "github.com/rclone/rclone/backend/storj"
-	_ "github.com/rclone/rclone/backend/sugarsync"
-	_ "github.com/rclone/rclone/backend/swift"
-	_ "github.com/rclone/rclone/backend/teldrive"
-	_ "github.com/rclone/rclone/backend/terabox"
-	_ "github.com/rclone/rclone/backend/ulozto"
-	_ "github.com/rclone/rclone/backend/union"
-	_ "github.com/rclone/rclone/backend/uptobox"
-	_ "github.com/rclone/rclone/backend/webdav"
-	_ "github.com/rclone/rclone/backend/yandex"
-	_ "github.com/rclone/rclone/backend/zoho"
+_ "github.com/rclone/rclone/backend/alias"
+_ "github.com/rclone/rclone/backend/alist"
+_ "github.com/rclone/rclone/backend/alldebrid"
+_ "github.com/rclone/rclone/backend/azureblob"
+_ "github.com/rclone/rclone/backend/azurefiles"
+_ "github.com/rclone/rclone/backend/b2"
+_ "github.com/rclone/rclone/backend/box"
+_ "github.com/rclone/rclone/backend/cache"
+_ "github.com/rclone/rclone/backend/chunker"
+_ "github.com/rclone/rclone/backend/cloudinary"
+_ "github.com/rclone/rclone/backend/combine"
+_ "github.com/rclone/rclone/backend/compress"
+_ "github.com/rclone/rclone/backend/crypt"
+_ "github.com/rclone/rclone/backend/doi"
+_ "github.com/rclone/rclone/backend/drive"
+_ "github.com/rclone/rclone/backend/dropbox"
+_ "github.com/rclone/rclone/backend/fichier"
+_ "github.com/rclone/rclone/backend/filefabric"
+_ "github.com/rclone/rclone/backend/filelu"
+_ "github.com/rclone/rclone/backend/filescom"
+_ "github.com/rclone/rclone/backend/ftp"
+_ "github.com/rclone/rclone/backend/gofile"
+_ "github.com/rclone/rclone/backend/googlecloudstorage"
+_ "github.com/rclone/rclone/backend/googlephotos"
+_ "github.com/rclone/rclone/backend/hasher"
+_ "github.com/rclone/rclone/backend/hdfs"
+_ "github.com/rclone/rclone/backend/hidrive"
+_ "github.com/rclone/rclone/backend/http"
+_ "github.com/rclone/rclone/backend/iclouddrive"
+_ "github.com/rclone/rclone/backend/imagekit"
+_ "github.com/rclone/rclone/backend/internetarchive"
+_ "github.com/rclone/rclone/backend/jottacloud"
+_ "github.com/rclone/rclone/backend/koofr"
+_ "github.com/rclone/rclone/backend/linkbox"
+_ "github.com/rclone/rclone/backend/local"
+_ "github.com/rclone/rclone/backend/mailru"
+_ "github.com/rclone/rclone/backend/mega"
+_ "github.com/rclone/rclone/backend/memory"
+_ "github.com/rclone/rclone/backend/netstorage"
+_ "github.com/rclone/rclone/backend/onedrive"
+_ "github.com/rclone/rclone/backend/opendrive"
+_ "github.com/rclone/rclone/backend/oracleobjectstorage"
+_ "github.com/rclone/rclone/backend/pcloud"
+_ "github.com/rclone/rclone/backend/pikpak"
+_ "github.com/rclone/rclone/backend/pixeldrain"
+_ "github.com/rclone/rclone/backend/premiumizeme"
+_ "github.com/rclone/rclone/backend/protondrive"
+_ "github.com/rclone/rclone/backend/putio"
+_ "github.com/rclone/rclone/backend/qingstor"
+_ "github.com/rclone/rclone/backend/quatrix"
+_ "github.com/rclone/rclone/backend/s3"
+_ "github.com/rclone/rclone/backend/seafile"
+_ "github.com/rclone/rclone/backend/sftp"
+_ "github.com/rclone/rclone/backend/sharefile"
+_ "github.com/rclone/rclone/backend/sia"
+_ "github.com/rclone/rclone/backend/smb"
+_ "github.com/rclone/rclone/backend/storj"
+_ "github.com/rclone/rclone/backend/sugarsync"
+_ "github.com/rclone/rclone/backend/swift"
+_ "github.com/rclone/rclone/backend/teldrive"
+_ "github.com/rclone/rclone/backend/terabox"
+_ "github.com/rclone/rclone/backend/ulozto"
+_ "github.com/rclone/rclone/backend/union"
+_ "github.com/rclone/rclone/backend/uptobox"
+_ "github.com/rclone/rclone/backend/webdav"
+_ "github.com/rclone/rclone/backend/yandex"
+_ "github.com/rclone/rclone/backend/zoho"
 )

backend/alldebrid/alldebrid.go

@@ -1,4 +1,4 @@
-// Package alldebrid provides an interface to the alldebrid.com
+// Package alldebrid provides an interface to the alldebrid.com
 // object storage system.
 package alldebrid
 

backend/alldebrid/alldebrid_test.go

@@ -1,4 +1,4 @@
-package alldebrid_test
+package alldebrid_test
 
 import (
 	"testing"

backend/alldebrid/api/types.go

@@ -1,4 +1,4 @@
-// Package api contains definitions for using the alldebrid API
+// Package api contains definitions for using the alldebrid API
 package api
 
 import "fmt"

backend/azureblob/azureblob.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !solaris && !js
+//go:build !plan9 && !solaris && !js
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
@@ -2152,7 +2152,6 @@ func (o *Object) getMetadata() (metadata map[string]*string) {
 	}
 	metadata = make(map[string]*string, len(o.meta))
 	for k, v := range o.meta {
-		v := v
 		metadata[k] = &v
 	}
 	return metadata
@@ -2798,8 +2797,6 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) {
 		blockList    blockblob.GetBlockListResponse
 		properties   *blob.GetPropertiesResponse
 		options      *blockblob.CommitBlockListOptions
-		// Use temporary pacer as this can be called recursively which can cause a deadlock with --max-connections
-		pacer = fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
 	)
 
 	properties, err = o.readMetaDataAlways(ctx)
@@ -2811,7 +2808,7 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) {
 
 	if objectExists {
 		// Get the committed block list
-		err = pacer.Call(func() (bool, error) {
+		err = o.fs.pacer.Call(func() (bool, error) {
 			blockList, err = blockBlobSVC.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
 			return o.fs.shouldRetry(ctx, err)
 		})
@@ -2853,7 +2850,7 @@ func (o *Object) clearUncommittedBlocks(ctx context.Context) (err error) {
 
 	// Commit only the committed blocks
 	fs.Debugf(o, "Committing %d blocks to remove uncommitted blocks", len(blockIDs))
-	err = pacer.Call(func() (bool, error) {
+	err = o.fs.pacer.Call(func() (bool, error) {
 		_, err := blockBlobSVC.CommitBlockList(ctx, blockIDs, options)
 		return o.fs.shouldRetry(ctx, err)
 	})

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !solaris && !js
+//go:build !plan9 && !solaris && !js
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,4 +1,4 @@
-// Test AzureBlob filesystem interface
+// Test AzureBlob filesystem interface
 
 //go:build !plan9 && !solaris && !js
 

backend/azureblob/azureblob_unsupported.go

@@ -1,4 +1,4 @@
-// Build for azureblob for unsupported platforms to stop go complaining
+// Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
 //go:build plan9 || solaris || js

backend/azurefiles/azurefiles.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !js
+//go:build !plan9 && !js
 
 // Package azurefiles provides an interface to Microsoft Azure Files
 package azurefiles
@@ -1313,10 +1313,29 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 	srcURL := srcObj.fileClient().URL()
 	fc := f.fileClient(remote)
-	_, err = fc.StartCopyFromURL(ctx, srcURL, &opt)
+	startCopy, err := fc.StartCopyFromURL(ctx, srcURL, &opt)
 	if err != nil {
 		return nil, fmt.Errorf("Copy failed: %w", err)
 	}
+
+	// Poll for completion if necessary
+	//
+	// The for loop is never executed for same storage account copies.
+	copyStatus := startCopy.CopyStatus
+	var properties file.GetPropertiesResponse
+	pollTime := 100 * time.Millisecond
+
+	for copyStatus != nil && string(*copyStatus) == string(file.CopyStatusTypePending) {
+		time.Sleep(pollTime)
+
+		properties, err = fc.GetProperties(ctx, &file.GetPropertiesOptions{})
+		if err != nil {
+			return nil, err
+		}
+		copyStatus = properties.CopyStatus
+		pollTime = min(2*pollTime, time.Second)
+	}
+
 	dstObj, err := f.NewObject(ctx, remote)
 	if err != nil {
 		return nil, fmt.Errorf("Copy: NewObject failed: %w", err)

backend/azurefiles/azurefiles_internal_test.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !js
+//go:build !plan9 && !js
 
 package azurefiles
 

backend/azurefiles/azurefiles_test.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !js
+//go:build !plan9 && !js
 
 package azurefiles
 

backend/azurefiles/azurefiles_unsupported.go

@@ -1,4 +1,4 @@
-// Build for azurefiles for unsupported platforms to stop go complaining
+// Build for azurefiles for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
 //go:build plan9 || js

backend/b2/api/types.go

@@ -1,4 +1,4 @@
-// Package api provides types used by the Backblaze B2 API.
+// Package api provides types used by the Backblaze B2 API.
 package api
 
 import (

backend/b2/api/types_test.go

@@ -1,4 +1,4 @@
-package api_test
+package api_test
 
 import (
 	"testing"

backend/b2/b2.go

@@ -1,4 +1,4 @@
-// Package b2 provides an interface to the Backblaze B2 object storage system.
+// Package b2 provides an interface to the Backblaze B2 object storage system.
 package b2
 
 // FIXME should we remove sha1 checks from here as rclone now supports
@@ -2224,13 +2224,17 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 		return info, nil, err
 	}
 
+	up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
+	if err != nil {
+		return info, nil, err
+	}
+
 	info = fs.ChunkWriterInfo{
-		ChunkSize:   int64(f.opt.ChunkSize),
+		ChunkSize:   up.chunkSize,
 		Concurrency: o.fs.opt.UploadConcurrency,
 		//LeavePartsOnError: o.fs.opt.LeavePartsOnError,
 	}
-	up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
-	return info, up, err
+	return info, up, nil
 }
 
 // Remove an object

backend/b2/b2_internal_test.go

@@ -1,4 +1,4 @@
-package b2
+package b2
 
 import (
 	"context"

backend/b2/b2_test.go

@@ -1,4 +1,4 @@
-// Test B2 filesystem interface
+// Test B2 filesystem interface
 package b2
 
 import (

backend/b2/upload.go

@@ -1,4 +1,4 @@
-// Upload large files for b2
+// Upload large files for b2
 //
 // Docs - https://www.backblaze.com/docs/cloud-storage-large-files
 

backend/box/api/types.go

@@ -1,4 +1,4 @@
-// Package api has type definitions for box
+// Package api has type definitions for box
 //
 // Converted from the API docs with help from https://mholt.github.io/json-to-go/
 package api

backend/box/box.go

@@ -1,4 +1,4 @@
-// Package box provides an interface to the Box
+// Package box provides an interface to the Box
 // object storage system.
 package box
 

backend/box/box_test.go

@@ -1,4 +1,4 @@
-// Test Box filesystem interface
+// Test Box filesystem interface
 package box_test
 
 import (