From c72f71bd028d5a18a36b5ca1a348152954a4f675 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 14 Feb 2025 13:02:54 +0000 Subject: [PATCH 01/73] Start v1.69.1-DEV development --- VERSION | 2 +- docs/layouts/partials/version.html | 2 +- fs/versiontag.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 6439851b65393..5f8853011028d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.69.0 +v1.69.1 diff --git a/docs/layouts/partials/version.html b/docs/layouts/partials/version.html index 294e240e96106..003a53d3b0e53 100644 --- a/docs/layouts/partials/version.html +++ b/docs/layouts/partials/version.html @@ -1 +1 @@ -v1.69.0 \ No newline at end of file +v1.69.1 \ No newline at end of file diff --git a/fs/versiontag.go b/fs/versiontag.go index 0080a00a56981..715d12ab0a0bd 100644 --- a/fs/versiontag.go +++ b/fs/versiontag.go @@ -1,4 +1,4 @@ package fs // VersionTag of rclone -var VersionTag = "v1.69.0" +var VersionTag = "v1.69.1" From ddaeb0701968ba622a362faad58256f04fa52cb8 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 13 Jan 2025 18:35:27 +0000 Subject: [PATCH 02/73] doc: make man page well formed for whatis - fixes #7430 --- bin/make_manual.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/bin/make_manual.py b/bin/make_manual.py index d2e756457431d..2d0fd580e6c5a 100755 --- a/bin/make_manual.py +++ b/bin/make_manual.py @@ -7,6 +7,7 @@ import os import re import time +import subprocess from datetime import datetime docpath = "docs/content" @@ -192,13 +193,23 @@ def main(): command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub build_date = datetime.utcfromtimestamp( int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))) + help_output = subprocess.check_output(["rclone", "help"]).decode("utf-8") with open(outfile, "w") as out: out.write("""\ %% rclone(1) User Manual %% Nick Craig-Wood %% %s -""" % build_date.strftime("%b %d, %Y")) +# NAME + +rclone - manage files on cloud storage + +# SYNOPSIS + +``` +%s +``` +""" % (build_date.strftime("%b %d, %Y"), help_output)) for doc in docs: contents = read_doc(doc) # Substitute the commands into doc.md From 376a5b1a836db9df21c72ca45bc0781011c2e6b3 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 28 Dec 2024 16:29:55 +0000 Subject: [PATCH 03/73] serve s3: fix list objects encoding-type Before this change rclone would always use encoding-type url even if the client hadn't asked for it. This confused some clients. This fixes the problem by leaving the URL encoding to the gofakes3 library which has also been fixed. Fixes #7836 --- cmd/serve/s3/backend.go | 4 ++-- cmd/serve/s3/list.go | 4 ++-- go.mod | 3 ++- go.sum | 6 ++++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go index ac0244dfdf78f..42671b11e5441 100644 --- a/cmd/serve/s3/backend.go +++ b/cmd/serve/s3/backend.go @@ -52,7 +52,7 @@ func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, err for _, entry := range dirEntries { if entry.IsDir() { response = append(response, gofakes3.BucketInfo{ - Name: gofakes3.URLEncode(entry.Name()), + Name: entry.Name(), CreationDate: gofakes3.NewContentTime(entry.ModTime()), }) } @@ -227,7 +227,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string } return &gofakes3.Object{ - Name: gofakes3.URLEncode(objectName), + Name: objectName, Hash: hash, Metadata: meta, Size: size, diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go index 1bec58d056360..554d152ab4062 100644 --- a/cmd/serve/s3/list.go +++ b/cmd/serve/s3/list.go @@ -28,7 +28,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucket, fdPath, name string, addPr if entry.IsDir() { if addPrefix { - response.AddPrefix(gofakes3.URLEncode(objectPath)) + response.AddPrefix(objectPath) continue } err := b.entryListR(_vfs, bucket, path.Join(fdPath, object), "", false, response) @@ -37,7 +37,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucket, fdPath, name string, addPr } } else { item := &gofakes3.Content{ - Key: gofakes3.URLEncode(objectPath), + Key: objectPath, LastModified: gofakes3.NewContentTime(entry.ModTime()), ETag: getFileHash(entry), Size: entry.Size(), diff --git a/go.mod b/go.mod index 962aaea79e310..f4d830b8925ec 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 github.com/quasilyte/go-ruleguard/dsl v0.3.22 - github.com/rclone/gofakes3 v0.0.3 + github.com/rclone/gofakes3 v0.0.4 github.com/rfjakob/eme v1.1.2 github.com/rivo/uniseg v0.4.7 github.com/rogpeppe/go-internal v1.12.0 @@ -177,6 +177,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/xxml v0.0.3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/panjf2000/ants/v2 v2.9.1 // indirect diff --git a/go.sum b/go.sum index 1b0f4578f4019..06a76c5ef9a23 100644 --- a/go.sum +++ b/go.sum @@ -461,6 +461,8 @@ github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tB github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/xxml v0.0.3 h1:ZIpPQpfyG5uZQnqqC0LZuWtPk/WT8G/qkxvO6jb7zMU= +github.com/minio/xxml v0.0.3/go.mod h1:wcXErosl6IezQIMEWSK/LYC2VS7LJ1dAkgvuyIN3aH4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= @@ -529,8 +531,8 @@ github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1 github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93 h1:UVArwN/wkKjMVhh2EQGC0tEc1+FqiLlvYXY5mQ2f8Wg= github.com/rasky/go-xdr v0.0.0-20170124162913-1a41d1a06c93/go.mod h1:Nfe4efndBz4TibWycNE+lqyJZiMX4ycx+QKV8Ta0f/o= -github.com/rclone/gofakes3 v0.0.3 h1:0sKCxJ8TUUAG5KXGuc/fcDKGnzB/j6IjNQui9ntIZPo= -github.com/rclone/gofakes3 v0.0.3/go.mod h1:z7+o2VUwitO0WuVHReQlOW9jZ03LpeJ0PUFSULyTIds= +github.com/rclone/gofakes3 v0.0.4 h1:LswpC49VY/UJ1zucoL5ktnOEX6lq3qK7e1aFIAfqCbk= +github.com/rclone/gofakes3 v0.0.4/go.mod h1:j/UoS+2/Mr7xAlfKhyVC58YyFQmh9uoQA5YZQXQUqmg= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= From 1ef2da31a7c3bed602124f7b049532fce0a59706 Mon Sep 17 00:00:00 2001 From: Matt Ickstadt Date: Tue, 14 Jan 2025 10:18:53 -0600 Subject: [PATCH 04/73] docs: fix link to Rclone Serve S3 --- docs/content/s3.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/s3.md b/docs/content/s3.md index 280f5dd6d3d2c..c0565a978ba53 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -31,7 +31,7 @@ The S3 backend can be used with a number of different providers: {{< provider name="Petabox" home="https://petabox.io/" config="/s3/#petabox" >}} {{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}} {{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}} -{{< provider name="Rclone Serve S3" home="/commands/rclone_serve_http/" config="/s3/#rclone" >}} +{{< provider name="Rclone Serve S3" home="/commands/rclone_serve_s3/" config="/s3/#rclone" >}} {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}} {{< provider name="Seagate Lyve Cloud" home="https://www.seagate.com/gb/en/services/cloud/storage/" config="/s3/#lyve" >}} {{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}} @@ -3728,7 +3728,7 @@ location_constraint = au-nsw ### Rclone Serve S3 {#rclone} Rclone can serve any remote over the S3 protocol. For details see the -[rclone serve s3](/commands/rclone_serve_http/) documentation. +[rclone serve s3](/commands/rclone_serve_s3/) documentation. For example, to serve `remote:path` over s3, run the server like this: @@ -3749,7 +3749,7 @@ use_multipart_uploads = false ``` Note that setting `disable_multipart_uploads = true` is to work around -[a bug](/commands/rclone_serve_http/#bugs) which will be fixed in due course. +[a bug](/commands/rclone_serve_s3/#bugs) which will be fixed in due course. ### Scaleway From 626bdacd59182621d5a868536b5d2211d8a752c8 Mon Sep 17 00:00:00 2001 From: Matt Ickstadt Date: Tue, 14 Jan 2025 10:37:25 -0600 Subject: [PATCH 05/73] docs: fix reference to serves3 setting disable_multipart_uploads which was renamed --- cmd/serve/s3/serve_s3.md | 2 +- docs/content/s3.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md index a7cda62326173..1c805ab4809c9 100644 --- a/cmd/serve/s3/serve_s3.md +++ b/cmd/serve/s3/serve_s3.md @@ -69,7 +69,7 @@ secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false ``` -Note that setting `disable_multipart_uploads = true` is to work around +Note that setting `use_multipart_uploads = false` is to work around [a bug](#bugs) which will be fixed in due course. ### Bugs diff --git a/docs/content/s3.md b/docs/content/s3.md index c0565a978ba53..fa1fbe2e57ae8 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -3748,7 +3748,7 @@ secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false ``` -Note that setting `disable_multipart_uploads = true` is to work around +Note that setting `use_multipart_uploads = false` is to work around [a bug](/commands/rclone_serve_s3/#bugs) which will be fixed in due course. ### Scaleway From 1b47b7a6bb77f92f48cd99520f3f51c446d64b4b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 14 Jan 2025 20:21:40 +0000 Subject: [PATCH 06/73] vfs: fix race detected by race detector This race would only happen when --dir-cache-time was very small. This was noticed in the VFS tests when --dir-cache-time was 100 mS so is unlikely to affect normal users. --- vfs/dir.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vfs/dir.go b/vfs/dir.go index da0f8d47c01f1..c72494eee80b3 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -66,7 +66,10 @@ func newDir(vfs *VFS, f fs.Fs, parent *Dir, fsDir fs.Directory) *Dir { inode: newInode(), items: make(map[string]Node), } - d.cleanupTimer = time.AfterFunc(time.Duration(vfs.Opt.DirCacheTime*2), d.cacheCleanup) + // Set timer up like this to avoid race of d.cacheCleanup being called + // before d.cleanupTimer is assigned to + d.cleanupTimer = time.AfterFunc(time.Hour, d.cacheCleanup) + d.cleanupTimer.Reset(time.Duration(vfs.Opt.DirCacheTime * 2)) return d } From 7988300f50ed34a8f86b68c3aa5575787f9b02be Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 15 Jan 2025 16:32:59 +0000 Subject: [PATCH 07/73] fs: fix confusing "didn't find section in config file" error This change decorates the error with the section name not found which will hopefully save user confusion. Fixes #8170 --- backend/googlephotos/googlephotos_test.go | 3 ++- fs/cache/cache_test.go | 4 ++-- fs/newfs.go | 3 ++- fs/rc/rcserver/rcserver_test.go | 2 +- fstest/fstests/fstests.go | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/googlephotos/googlephotos_test.go b/backend/googlephotos/googlephotos_test.go index cd1e0943f8411..0eeb99bcebe91 100644 --- a/backend/googlephotos/googlephotos_test.go +++ b/backend/googlephotos/googlephotos_test.go @@ -2,6 +2,7 @@ package googlephotos import ( "context" + "errors" "fmt" "io" "net/http" @@ -35,7 +36,7 @@ func TestIntegration(t *testing.T) { *fstest.RemoteName = "TestGooglePhotos:" } f, err := fs.NewFs(ctx, *fstest.RemoteName) - if err == fs.ErrorNotFoundInConfigFile { + if errors.Is(err, fs.ErrorNotFoundInConfigFile) { t.Skipf("Couldn't create google photos backend - skipping tests: %v", err) } require.NoError(t, err) diff --git a/fs/cache/cache_test.go b/fs/cache/cache_test.go index d9d54fa2e13c3..0448f02440e2b 100644 --- a/fs/cache/cache_test.go +++ b/fs/cache/cache_test.go @@ -131,7 +131,7 @@ func TestPutErr(t *testing.T) { assert.Equal(t, 1, Entries()) fNew, err := GetFn(context.Background(), "mock:/", create) - require.Equal(t, fs.ErrorNotFoundInConfigFile, err) + require.True(t, errors.Is(err, fs.ErrorNotFoundInConfigFile)) require.Equal(t, f, fNew) assert.Equal(t, 1, Entries()) @@ -141,7 +141,7 @@ func TestPutErr(t *testing.T) { PutErr("mock:/file.txt", f, fs.ErrorNotFoundInConfigFile) fNew, err = GetFn(context.Background(), "mock:/file.txt", create) - require.Equal(t, fs.ErrorNotFoundInConfigFile, err) + require.True(t, errors.Is(err, fs.ErrorNotFoundInConfigFile)) require.Equal(t, f, fNew) assert.Equal(t, 1, Entries()) diff --git a/fs/newfs.go b/fs/newfs.go index d59a505a64eb2..f4f7dd346b28c 100644 --- a/fs/newfs.go +++ b/fs/newfs.go @@ -6,6 +6,7 @@ import ( "context" "crypto/md5" "encoding/base64" + "fmt" "os" "path/filepath" "strings" @@ -104,7 +105,7 @@ func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, conne m := ConfigMap("", nil, configName, parsed.Config) fsName, ok = m.Get("type") if !ok { - return nil, "", "", nil, ErrorNotFoundInConfigFile + return nil, "", "", nil, fmt.Errorf("%w (%q)", ErrorNotFoundInConfigFile, configName) } } } else { diff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.go index 29738b614fc24..f79a926cff582 100644 --- a/fs/rc/rcserver/rcserver_test.go +++ b/fs/rc/rcserver/rcserver_test.go @@ -358,7 +358,7 @@ func TestRemoteServing(t *testing.T) { URL: "[notfoundremote:]/", Status: http.StatusInternalServerError, Expected: `{ - "error": "failed to make Fs: didn't find section in config file", + "error": "failed to make Fs: didn't find section in config file (\"notfoundremote\")", "input": null, "path": "/", "status": 500 diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 2e840cd0c3baa..6024177c65e5c 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -459,7 +459,7 @@ func Run(t *testing.T, opt *Opt) { subRemoteName, subRemoteLeaf, err = fstest.RandomRemoteName(remoteName) require.NoError(t, err) f, err = fs.NewFs(context.Background(), subRemoteName) - if err == fs.ErrorNotFoundInConfigFile { + if errors.Is(err, fs.ErrorNotFoundInConfigFile) { t.Logf("Didn't find %q in config file - skipping tests", remoteName) return } From 6cbb9fd7cbfb500517c2c78e7833e0b4ef4fd952 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:01:06 +0100 Subject: [PATCH 08/73] docs: fix typos found by codespell in docs and code comments --- backend/azurefiles/azurefiles.go | 2 +- backend/iclouddrive/iclouddrive.go | 2 +- backend/imagekit/client/media.go | 2 +- backend/pcloud/pcloud.go | 2 +- backend/s3/s3.go | 4 ++-- bin/not-in-stable.go | 10 +++++----- cmd/bisync/compare.go | 2 +- cmd/bisync/listing.go | 2 +- cmd/bisync/testdata/test_resync/golden/test.log | 2 +- cmd/bisync/testdata/test_resync/scenario.txt | 4 ++-- cmd/config/config.go | 4 ++-- cmd/copyurl/copyurl.go | 2 +- cmd/serve/nfs/filesystem.go | 2 +- cmd/serve/nfs/nfs.go | 2 +- cmd/serve/s3/backend.go | 4 ++-- cmd/serve/s3/list.go | 2 +- docs/content/bisync.md | 12 ++++++------ docs/content/crypt.md | 2 +- docs/content/docs.md | 2 +- docs/content/googlephotos.md | 2 +- docs/content/rc.md | 2 +- docs/content/s3.md | 2 +- fs/fshttp/http_test.go | 2 +- fs/log.go | 2 +- fs/object/object.go | 2 +- fs/object/object_test.go | 2 +- fstest/fstests/fstests.go | 2 +- vfs/rc.go | 2 +- vfs/vfs.go | 2 +- vfs/vfscache/item_test.go | 2 +- 30 files changed, 43 insertions(+), 43 deletions(-) diff --git a/backend/azurefiles/azurefiles.go b/backend/azurefiles/azurefiles.go index acff5be19fd2d..7d9a53075591a 100644 --- a/backend/azurefiles/azurefiles.go +++ b/backend/azurefiles/azurefiles.go @@ -899,7 +899,7 @@ func (o *Object) getMetadata(ctx context.Context) error { // Hash returns the MD5 of an object returning a lowercase hex string // -// May make a network request becaue the [fs.List] method does not +// May make a network request because the [fs.List] method does not // return MD5 hashes for DirEntry func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) { if ty != hash.MD5 { diff --git a/backend/iclouddrive/iclouddrive.go b/backend/iclouddrive/iclouddrive.go index 591d817182069..7228db0609874 100644 --- a/backend/iclouddrive/iclouddrive.go +++ b/backend/iclouddrive/iclouddrive.go @@ -445,7 +445,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, } // build request - // cant use normal rename as file needs to be "activated" first + // can't use normal rename as file needs to be "activated" first r := api.NewUpdateFileInfo() r.DocumentID = doc.DocumentID diff --git a/backend/imagekit/client/media.go b/backend/imagekit/client/media.go index d495b713b152e..1d4ee030bd2db 100644 --- a/backend/imagekit/client/media.go +++ b/backend/imagekit/client/media.go @@ -75,7 +75,7 @@ type MoveFolderParam struct { DestinationPath string `validate:"nonzero" json:"destinationPath"` } -// JobIDResponse respresents response struct with JobID for folder operations +// JobIDResponse represents response struct with JobID for folder operations type JobIDResponse struct { JobID string `json:"jobId"` } diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go index f198c556383f7..763784645c696 100644 --- a/backend/pcloud/pcloud.go +++ b/backend/pcloud/pcloud.go @@ -424,7 +424,7 @@ func (f *Fs) newSingleConnClient(ctx context.Context) (*rest.Client, error) { }) // Set our own http client in the context ctx = oauthutil.Context(ctx, baseClient) - // create a new oauth client, re-use the token source + // create a new oauth client, reuse the token source oAuthClient := oauth2.NewClient(ctx, f.ts) return rest.NewClient(oAuthClient).SetRoot("https://" + f.opt.Hostname), nil } diff --git a/backend/s3/s3.go b/backend/s3/s3.go index a0eda19ab23ae..d487d9404ba1a 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -3344,7 +3344,7 @@ func setQuirks(opt *Options) { listObjectsV2 = true // Always use ListObjectsV2 instead of ListObjects virtualHostStyle = true // Use bucket.provider.com instead of putting the bucket in the URL urlEncodeListings = true // URL encode the listings to help with control characters - useMultipartEtag = true // Set if Etags for multpart uploads are compatible with AWS + useMultipartEtag = true // Set if Etags for multipart uploads are compatible with AWS useAcceptEncodingGzip = true // Set Accept-Encoding: gzip mightGzip = true // assume all providers might use content encoding gzip until proven otherwise useAlreadyExists = true // Set if provider returns AlreadyOwnedByYou or no error if you try to remake your own bucket @@ -6057,7 +6057,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn if mOut == nil { err = fserrors.RetryErrorf("internal error: no info from multipart upload") } else if mOut.UploadId == nil { - err = fserrors.RetryErrorf("internal error: no UploadId in multpart upload: %#v", *mOut) + err = fserrors.RetryErrorf("internal error: no UploadId in multipart upload: %#v", *mOut) } } return f.shouldRetry(ctx, err) diff --git a/bin/not-in-stable.go b/bin/not-in-stable.go index 57a0257600b43..5f988af8271c2 100644 --- a/bin/not-in-stable.go +++ b/bin/not-in-stable.go @@ -29,7 +29,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) { cmd := exec.Command("git", "log", "--oneline", from+".."+to) out, err := cmd.Output() if err != nil { - log.Fatalf("failed to run git log %s: %v", from+".."+to, err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log. + log.Fatalf("failed to run git log %s: %v", from+".."+to, err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log. } logMap = map[string]string{} logs = []string{} @@ -39,7 +39,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) { } match := logRe.FindSubmatch(line) if match == nil { - log.Fatalf("failed to parse line: %q", line) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log. + log.Fatalf("failed to parse line: %q", line) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log. } var hash, logMessage = string(match[1]), string(match[2]) logMap[logMessage] = hash @@ -52,12 +52,12 @@ func main() { flag.Parse() args := flag.Args() if len(args) != 0 { - log.Fatalf("Syntax: %s", os.Args[0]) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log. + log.Fatalf("Syntax: %s", os.Args[0]) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log. } // v1.54.0 versionBytes, err := os.ReadFile("VERSION") if err != nil { - log.Fatalf("Failed to read version: %v", err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log. + log.Fatalf("Failed to read version: %v", err) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log. } if versionBytes[0] == 'v' { versionBytes = versionBytes[1:] @@ -65,7 +65,7 @@ func main() { versionBytes = bytes.TrimSpace(versionBytes) semver := semver.New(string(versionBytes)) stable := fmt.Sprintf("v%d.%d", semver.Major, semver.Minor-1) - log.Printf("Finding commits in %v not in stable %s", semver, stable) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. intead of log. + log.Printf("Finding commits in %v not in stable %s", semver, stable) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid ruleguard suggesting fs. instead of log. masterMap, masterLogs := readCommits(stable+".0", "master") stableMap, _ := readCommits(stable+".0", stable+"-stable") for _, logMessage := range masterLogs { diff --git a/cmd/bisync/compare.go b/cmd/bisync/compare.go index 75eac3b304fbf..774875dbb1580 100644 --- a/cmd/bisync/compare.go +++ b/cmd/bisync/compare.go @@ -218,7 +218,7 @@ func (b *bisyncRun) setFromCompareFlag(ctx context.Context) error { if b.opt.CompareFlag == "" { return nil } - var CompareFlag CompareOpt // for exlcusions + var CompareFlag CompareOpt // for exclusions opts := strings.Split(b.opt.CompareFlag, ",") for _, opt := range opts { switch strings.ToLower(strings.TrimSpace(opt)) { diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go index 13179b26f106e..7451a5cbb4dc9 100644 --- a/cmd/bisync/listing.go +++ b/cmd/bisync/listing.go @@ -394,7 +394,7 @@ func parseHash(str string) (string, string, error) { return "", "", fmt.Errorf("invalid hash %q", str) } -// checkListing verifies that listing is not empty (unless resynching) +// checkListing verifies that listing is not empty (unless resyncing) func (b *bisyncRun) checkListing(ls *fileList, listing, msg string) error { if b.opt.Resync || !ls.empty() { return nil diff --git a/cmd/bisync/testdata/test_resync/golden/test.log b/cmd/bisync/testdata/test_resync/golden/test.log index f9488a672573b..13a3bddcb6e0a 100644 --- a/cmd/bisync/testdata/test_resync/golden/test.log +++ b/cmd/bisync/testdata/test_resync/golden/test.log @@ -23,7 +23,7 @@ INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" INFO : Bisync successful (05) : move-listings empty-path1 -(06) : test 2. resync with empty path2, resulting in synching all content to path2. +(06) : test 2. resync with empty path2, resulting in syncing all content to path2. (07) : purge-children {path2/} (08) : bisync resync INFO : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set. diff --git a/cmd/bisync/testdata/test_resync/scenario.txt b/cmd/bisync/testdata/test_resync/scenario.txt index 078254646fa85..4a0e864752264 100644 --- a/cmd/bisync/testdata/test_resync/scenario.txt +++ b/cmd/bisync/testdata/test_resync/scenario.txt @@ -1,6 +1,6 @@ test resync # 1. Resync with empty Path1, resulting in copying all content FROM Path2 -# 2. Resync with empty Path2, resulting in synching all content TO Path2 +# 2. Resync with empty Path2, resulting in syncing all content TO Path2 # 3. Exercise all of the various file difference scenarios during a resync: # File Path1 Path2 Expected action Who wins # - file1.txt Exists Missing Sync Path1 >Path2 Path1 @@ -17,7 +17,7 @@ purge-children {path1/} bisync resync move-listings empty-path1 -test 2. resync with empty path2, resulting in synching all content to path2. +test 2. resync with empty path2, resulting in syncing all content to path2. purge-children {path2/} bisync resync move-listings empty-path2 diff --git a/cmd/config/config.go b/cmd/config/config.go index 4aefe62c02834..90630a9bca274 100644 --- a/cmd/config/config.go +++ b/cmd/config/config.go @@ -549,12 +549,12 @@ password to re-encrypt the config. When |--password-command| is called to change the password then the environment variable |RCLONE_PASSWORD_CHANGE=1| will be set. So if -changing passwords programatically you can use the environment +changing passwords programmatically you can use the environment variable to distinguish which password you must supply. Alternatively you can remove the password first (with |rclone config encryption remove|), then set it again with this command which may be -easier if you don't mind the unecrypted config file being on the disk +easier if you don't mind the unencrypted config file being on the disk briefly. `, "|", "`"), RunE: func(command *cobra.Command, args []string) error { diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go index 2a615de28977b..c5f46cbc70078 100644 --- a/cmd/copyurl/copyurl.go +++ b/cmd/copyurl/copyurl.go @@ -54,7 +54,7 @@ destination if there is one with the same name. Setting |--stdout| or making the output file name |-| will cause the output to be written to standard output. -### Troublshooting +### Troubleshooting If you can't get |rclone copyurl| to work then here are some things you can try: diff --git a/cmd/serve/nfs/filesystem.go b/cmd/serve/nfs/filesystem.go index 15e786c49c993..fe30326cb3846 100644 --- a/cmd/serve/nfs/filesystem.go +++ b/cmd/serve/nfs/filesystem.go @@ -194,7 +194,7 @@ func (f *FS) Chown(name string, uid, gid int) (err error) { return file.Chown(uid, gid) } -// Chtimes changes the acces time and modified time +// Chtimes changes the access time and modified time func (f *FS) Chtimes(name string, atime time.Time, mtime time.Time) (err error) { defer log.Trace(name, "atime=%v, mtime=%v", atime, mtime)("err=%v", &err) return f.vfs.Chtimes(name, atime, mtime) diff --git a/cmd/serve/nfs/nfs.go b/cmd/serve/nfs/nfs.go index acbe943f59dc2..ad94c5a496575 100644 --- a/cmd/serve/nfs/nfs.go +++ b/cmd/serve/nfs/nfs.go @@ -145,7 +145,7 @@ that it uses an on disk cache, but the cache entries are held as symlinks. Rclone will use the handle of the underlying file as the NFS handle which improves performance. This sort of cache can't be backed up and restored as the underlying handles will change. This is Linux -only. It requres running rclone as root or with |CAP_DAC_READ_SEARCH|. +only. It requires running rclone as root or with |CAP_DAC_READ_SEARCH|. You can run rclone with this extra permission by doing this to the rclone binary |sudo setcap cap_dac_read_search+ep /path/to/rclone|. diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go index 42671b11e5441..33ff6387b96d5 100644 --- a/cmd/serve/s3/backend.go +++ b/cmd/serve/s3/backend.go @@ -158,7 +158,7 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin }, nil } -// GetObject fetchs the object from the filesystem. +// GetObject fetches the object from the filesystem. func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) { _vfs, err := b.s.getVFS(ctx) if err != nil { @@ -400,7 +400,7 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str } fp := path.Join(bucketName, objectName) - // S3 does not report an error when attemping to delete a key that does not exist, so + // S3 does not report an error when attempting to delete a key that does not exist, so // we need to skip IsNotExist errors. if err := _vfs.Remove(fp); err != nil && !os.IsNotExist(err) { return err diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go index 554d152ab4062..9b0a2e11ed234 100644 --- a/cmd/serve/s3/list.go +++ b/cmd/serve/s3/list.go @@ -19,7 +19,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucket, fdPath, name string, addPr for _, entry := range dirEntries { object := entry.Name() - // workround for control-chars detect + // workaround for control-chars detect objectPath := path.Join(fdPath, object) if !strings.HasPrefix(object, name) { diff --git a/docs/content/bisync.md b/docs/content/bisync.md index 94a51fd9e054c..458f92eab7246 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -487,7 +487,7 @@ See the [bisync filters](#filtering) section and generic [--filter-from](/filtering/#filter-from-read-filtering-patterns-from-a-file) documentation. An [example filters file](#example-filters-file) contains filters for -non-allowed files for synching with Dropbox. +non-allowed files for syncing with Dropbox. If you make changes to your filters file then bisync requires a run with `--resync`. This is a safety feature, which prevents existing files @@ -664,7 +664,7 @@ Using `--check-sync=false` will disable it and may significantly reduce the sync run times for very large numbers of files. The check may be run manually with `--check-sync=only`. It runs only the -integrity check and terminates without actually synching. +integrity check and terminates without actually syncing. Note that currently, `--check-sync` **only checks listing snapshots and NOT the actual files on the remotes.** Note also that the listing snapshots will not @@ -1141,7 +1141,7 @@ The `--include*`, `--exclude*`, and `--filter` flags are also supported. ### How to filter directories -Filtering portions of the directory tree is a critical feature for synching. +Filtering portions of the directory tree is a critical feature for syncing. Examples of directory trees (always beneath the Path1/Path2 root level) you may want to exclude from your sync: @@ -1250,7 +1250,7 @@ quashed by adding `--quiet` to the bisync command line. ## Example exclude-style filters files for use with Dropbox {#exclude-filters} -- Dropbox disallows synching the listed temporary and configuration/data files. +- Dropbox disallows syncing the listed temporary and configuration/data files. The `- ` filters exclude these files where ever they may occur in the sync tree. Consider adding similar exclusions for file types you don't need to sync, such as core dump and software build files. @@ -1584,7 +1584,7 @@ test command flags can be equally prefixed by a single `-` or double dash. - `go test . -case basic -remote local -remote2 local` runs the `test_basic` test case using only the local filesystem, - synching one local directory with another local directory. + syncing one local directory with another local directory. Test script output is to the console, while commands within scenario.txt have their output sent to the `.../workdir/test.log` file, which is finally compared to the golden copy. @@ -1860,4 +1860,4 @@ causing bisync to consider more files than necessary due to overbroad filters du * Added [new `--ignore-listing-checksum` flag](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20%2D%2Dignore%2Dchecksum%20should%20be%20split%20into%20two%20flags%20for%20separate%20purposes) to distinguish from `--ignore-checksum` * [Performance improvements](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=6.%20Deletes%20take%20several%20times%20longer%20than%20copies) for large remotes -* Documentation and testing improvements \ No newline at end of file +* Documentation and testing improvements diff --git a/docs/content/crypt.md b/docs/content/crypt.md index 2ea2bf7d5f4e4..db3edcc74fa02 100644 --- a/docs/content/crypt.md +++ b/docs/content/crypt.md @@ -741,7 +741,7 @@ strong random number generator. The nonce is incremented for each chunk read making sure each nonce is unique for each block written. The chance of a nonce being reused is minuscule. If you wrote an exabyte of data (10¹⁸ bytes) you would have a probability of -approximately 2×10⁻³² of re-using a nonce. +approximately 2×10⁻³² of reusing a nonce. #### Chunk diff --git a/docs/content/docs.md b/docs/content/docs.md index dccb5c5f7ae6b..8b221f1307254 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -2930,7 +2930,7 @@ so they take exactly the same form. The options set by environment variables can be seen with the `-vv` flag, e.g. `rclone version -vv`. Options that can appear multiple times (type `stringArray`) are -treated slighly differently as environment variables can only be +treated slightly differently as environment variables can only be defined once. In order to allow a simple mechanism for adding one or many items, the input is treated as a [CSV encoded](https://godoc.org/encoding/csv) string. For example diff --git a/docs/content/googlephotos.md b/docs/content/googlephotos.md index 26faedcc6b7f1..c3549d1589053 100644 --- a/docs/content/googlephotos.md +++ b/docs/content/googlephotos.md @@ -384,7 +384,7 @@ Use the gphotosdl proxy for downloading the full resolution images The Google API will deliver images and video which aren't full resolution, and/or have EXIF data missing. -However if you ue the gphotosdl proxy tnen you can download original, +However if you use the gphotosdl proxy then you can download original, unchanged images. This runs a headless browser in the background. diff --git a/docs/content/rc.md b/docs/content/rc.md index 541093bb9fb15..4f54bb1700281 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -2068,7 +2068,7 @@ the `--vfs-cache-mode` is off, it will return an empty result. ], } -The `expiry` time is the time until the file is elegible for being +The `expiry` time is the time until the file is eligible for being uploaded in floating point seconds. This may go negative. As rclone only transfers `--transfers` files at once, only the lowest `--transfers` expiry times will have `uploading` as `true`. So there diff --git a/docs/content/s3.md b/docs/content/s3.md index fa1fbe2e57ae8..a71549e94be0b 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -750,7 +750,7 @@ Notes on above: that `USER_NAME` has been created. 2. The Resource entry must include both resource ARNs, as one implies the bucket and the other implies the bucket's objects. -3. When using [s3-no-check-bucket](#s3-no-check-bucket) and the bucket already exsits, the `"arn:aws:s3:::BUCKET_NAME"` doesn't have to be included. +3. When using [s3-no-check-bucket](#s3-no-check-bucket) and the bucket already exists, the `"arn:aws:s3:::BUCKET_NAME"` doesn't have to be included. For reference, [here's an Ansible script](https://gist.github.com/ebridges/ebfc9042dd7c756cd101cfa807b7ae2b) that will generate one or more buckets that will work with `rclone sync`. diff --git a/fs/fshttp/http_test.go b/fs/fshttp/http_test.go index 2c10e84a2a7d7..18f191e73a05c 100644 --- a/fs/fshttp/http_test.go +++ b/fs/fshttp/http_test.go @@ -133,7 +133,7 @@ func TestCertificates(t *testing.T) { assert.Fail(t, "Certificate expired", "Certificate expires at %s, current time is %s", cert[0].NotAfter.Sub(startTime), time.Since(startTime)) } - // Write some test data to fullfil the request + // Write some test data to fulfill the request w.Header().Set("Content-Type", "text/plain") _, _ = fmt.Fprintln(w, "test data") })) diff --git a/fs/log.go b/fs/log.go index 4b3abf6111d98..5088d1600466b 100644 --- a/fs/log.go +++ b/fs/log.go @@ -95,7 +95,7 @@ func LogValueHide(key string, value interface{}) LogValueItem { return LogValueItem{key: key, value: value, render: false} } -// String returns the representation of value. If render is fals this +// String returns the representation of value. If render is false this // is an empty string so LogValueItem entries won't show in the // textual representation of logs. func (j LogValueItem) String() string { diff --git a/fs/object/object.go b/fs/object/object.go index a8346edee5bbf..40aee8dec279b 100644 --- a/fs/object/object.go +++ b/fs/object/object.go @@ -297,7 +297,7 @@ func (o *MemoryObject) Open(ctx context.Context, options ...fs.OpenOption) (io.R // Update in to the object with the modTime given of the given size // -// This re-uses the internal buffer if at all possible. +// This reuses the internal buffer if at all possible. func (o *MemoryObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) { size := src.Size() if size == 0 { diff --git a/fs/object/object_test.go b/fs/object/object_test.go index 64bb361401347..2a423627281b6 100644 --- a/fs/object/object_test.go +++ b/fs/object/object_test.go @@ -142,7 +142,7 @@ func TestMemoryObject(t *testing.T) { assert.NoError(t, err) checkContent(o, "Rutabaga") assert.Equal(t, newNow, o.ModTime(context.Background())) - assert.Equal(t, "Rutaba", string(content)) // check we re-used the buffer + assert.Equal(t, "Rutaba", string(content)) // check we reused the buffer // not within the buffer newStr := "0123456789" diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go index 6024177c65e5c..2369e7c2dae66 100644 --- a/fstest/fstests/fstests.go +++ b/fstest/fstests/fstests.go @@ -2391,7 +2391,7 @@ func Run(t *testing.T, opt *Opt) { var itemCopy = item itemCopy.Path += ".copy" - // Set copy cutoff to mininum value so we make chunks + // Set copy cutoff to minimum value so we make chunks origCutoff, err := do.SetCopyCutoff(minChunkSize) require.NoError(t, err) defer func() { diff --git a/vfs/rc.go b/vfs/rc.go index 5c74e9f91e174..c2ad3caca85f0 100644 --- a/vfs/rc.go +++ b/vfs/rc.go @@ -464,7 +464,7 @@ the |--vfs-cache-mode| is off, it will return an empty result. ], } -The |expiry| time is the time until the file is elegible for being +The |expiry| time is the time until the file is eligible for being uploaded in floating point seconds. This may go negative. As rclone only transfers |--transfers| files at once, only the lowest |--transfers| expiry times will have |uploading| as |true|. So there diff --git a/vfs/vfs.go b/vfs/vfs.go index aa6fe15eb8a08..68b17f483ae46 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -216,7 +216,7 @@ func New(f fs.Fs, opt *vfscommon.Options) *VFS { configName := fs.ConfigString(f) for _, activeVFS := range active[configName] { if vfs.Opt == activeVFS.Opt { - fs.Debugf(f, "Re-using VFS from active cache") + fs.Debugf(f, "Reusing VFS from active cache") activeVFS.inUse.Add(1) return activeVFS } diff --git a/vfs/vfscache/item_test.go b/vfs/vfscache/item_test.go index 9fa2f3dbeeac0..fa5840e010685 100644 --- a/vfs/vfscache/item_test.go +++ b/vfs/vfscache/item_test.go @@ -428,7 +428,7 @@ func TestItemReloadCacheStale(t *testing.T) { assert.Equal(t, int64(110), fi.Size()) // Write to the file to make it dirty - // This checks we aren't re-using stale data + // This checks we aren't reusing stale data n, err := item.WriteAt([]byte("HELLO"), 0) require.NoError(t, err) assert.Equal(t, 5, n) From ad941655c54c0f329d360583f0acc0528d145e99 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 15 Jan 2025 17:58:49 +0000 Subject: [PATCH 09/73] iclouddrive: add notes on ADP and Missing PCS cookies - fixes #8310 --- docs/content/iclouddrive.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/content/iclouddrive.md b/docs/content/iclouddrive.md index 63023969b9f27..299cb064a3be8 100644 --- a/docs/content/iclouddrive.md +++ b/docs/content/iclouddrive.md @@ -61,7 +61,7 @@ Enter a value. config_2fa> 2FACODE Remote config -------------------- -[koofr] +[iclouddrive] - type: iclouddrive - apple_id: APPLEID - password: *** ENCRYPTED *** @@ -78,6 +78,20 @@ y/e/d> y ADP is currently unsupported and need to be disabled +On iPhone, Settings `>` Apple Account `>` iCloud `>` 'Access iCloud Data on the Web' must be ON, and 'Advanced Data Protection' OFF. + +## Troubleshooting + +### Missing PCS cookies from the request + +This means you have Advanced Data Protection (ADP) turned on. This is not supported at the moment. If you want to use rclone you will have to turn it off. See above for how to turn it off. + +You will need to clear the `cookies` and the `trust_token` fields in the config. Or you can delete the remote config and start again. + +You should then run `rclone reconnect remote:`. + +Note that changing the ADP setting may not take effect immediately - you may need to wait a few hours or a day before you can get rclone to work - keep clearing the config entry and running `rclone reconnect remote:` until rclone functions properly. + {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/iclouddrive/iclouddrive.go then run make backenddocs" >}} ### Standard options From de69448565115b2f1f3c612cd1c407ab26dccbfd Mon Sep 17 00:00:00 2001 From: Christoph Berger Date: Fri, 17 Jan 2025 09:50:22 +0100 Subject: [PATCH 10/73] docs: explain the stringArray flag parameter descriptor --- docs/content/docs.md | 5 +++++ docs/content/flags.md | 2 ++ 2 files changed, 7 insertions(+) diff --git a/docs/content/docs.md b/docs/content/docs.md index 8b221f1307254..17c909fea20f7 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -619,6 +619,11 @@ it to `false`. It is also possible to specify `--boolean=false` or parsed as `--boolean` and the `false` is parsed as an extra command line argument for rclone. +Options documented to take a `stringArray` parameter accept multiple +values. To pass more than one value, repeat the option; for example: +`--include value1 --include value2`. + + ### Time or duration options {#time-option} TIME or DURATION options can be specified as a duration string or a diff --git a/docs/content/flags.md b/docs/content/flags.md index aefd4dc6443b1..f9d32c490d527 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -9,6 +9,8 @@ description: "Rclone Global Flags" This describes the global flags available to every rclone command split into groups. +See the [Options section](/docs/#options) for syntax and usage advice. + ## Copy From 64556d4ca226435a5dde3c4dc8bcad58c284af7f Mon Sep 17 00:00:00 2001 From: Tim White Date: Wed, 15 Jan 2025 14:17:17 +0800 Subject: [PATCH 11/73] docs: add OneDrive Impersonate instructions - fixes #5610 --- docs/content/onedrive.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index a14b7a79e3ee2..91c8520831d1a 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -936,6 +936,28 @@ See the [metadata](/docs/#metadata) docs for more info. {{< rem autogenerated options stop >}} +### Impersonate other users as Admin + +Unlike Google Drive and impersonating any domain user via service accounts, OneDrive requires you to authenticate as an admin account, and manually setup a remote per user you wish to impersonate. + +1. In [Microsoft 365 Admin Center](https://admin.microsoft.com), open each user you need to "impersonate" and go to the OneDrive section. There is a heading called "Get access to files", you need to click to create the link, this creates the link of the format `https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/` but also changes the permissions so you your admin user has access. +2. Then in powershell run the following commands: +```console +Install-Module Microsoft.Graph -Scope CurrentUser -Repository PSGallery -Force +Import-Module Microsoft.Graph.Files +Connect-MgGraph -Scopes "Files.ReadWrite.All" +# Follow the steps to allow access to your admin user +# Then run this for each user you want to impersonate to get the Drive ID +Get-MgUserDefaultDrive -UserId '{emailaddress}' +# This will give you output of the format: +# Name Id DriveType CreatedDateTime +# ---- -- --------- --------------- +# OneDrive b!XYZ123 business 14/10/2023 1:00:58 pm + +``` +3. Then in rclone add a onedrive remote type, and use the `Type in driveID` with the DriveID you got in the previous step. One remote per user. It will then confirm the drive ID, and hopefully give you a message of `Found drive "root" of type "business"` and then include the URL of the format `https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/Documents` + + ## Limitations If you don't use rclone for 90 days the refresh token will From 642d1415d15d311a46286b489d7a4403e6fed71d Mon Sep 17 00:00:00 2001 From: izouxv Date: Fri, 17 Jan 2025 23:38:09 +0800 Subject: [PATCH 12/73] vfs: close the change notify channel on Shutdown --- vfs/vfs.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vfs/vfs.go b/vfs/vfs.go index 68b17f483ae46..8907a6e507828 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -365,6 +365,11 @@ func (vfs *VFS) Shutdown() { activeMu.Unlock() vfs.shutdownCache() + + if vfs.pollChan != nil { + close(vfs.pollChan) + vfs.pollChan = nil + } } // CleanUp deletes the contents of the on disk cache From c4b592e5498b56ff0592f86acc99a124d0911dbd Mon Sep 17 00:00:00 2001 From: Bruno Fernandes Date: Tue, 15 Oct 2024 16:18:33 -0300 Subject: [PATCH 13/73] s3: Added new storage class to magalu provider --- backend/s3/s3.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index d487d9404ba1a..c26d63a3cb703 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1343,7 +1343,7 @@ func init() { }, { Name: "endpoint", Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.", - Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,MagaluCloud,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox", + Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox", Examples: []fs.OptionExample{{ Value: "objects-us-east-1.dream.io", Help: "Dream Objects endpoint", @@ -1476,14 +1476,6 @@ func init() { Value: "s3.ir-tbz-sh1.arvanstorage.ir", Help: "ArvanCloud Tabriz Iran (Shahriar) endpoint", Provider: "ArvanCloud", - }, { - Value: "br-se1.magaluobjects.com", - Help: "Magalu BR Southeast 1 endpoint", - Provider: "Magalu", - }, { - Value: "br-ne1.magaluobjects.com", - Help: "Magalu BR Northeast 1 endpoint", - Provider: "Magalu", }}, }, { Name: "location_constraint", @@ -2122,13 +2114,16 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke Help: "Standard storage class", }}, }, { - // Mapping from here: #todo + // Mapping from here: https://docs.magalu.cloud/docs/storage/object-storage/Classes-de-Armazenamento/standard Name: "storage_class", Help: "The storage class to use when storing new objects in Magalu.", Provider: "Magalu", Examples: []fs.OptionExample{{ Value: "STANDARD", Help: "Standard storage class", + }, { + Value: "GLACIER_IR", + Help: "Glacier Instant Retrieval storage class", }}, }, { // Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925 From 1a95a23fdcb50136859eac589025e2400a3dbc6d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 22 Jan 2025 10:59:57 +0000 Subject: [PATCH 14/73] onedrive: mark German (de) region as deprecated See: https://learn.microsoft.com/en-us/previous-versions/azure/germany/ --- backend/onedrive/onedrive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index 980eec8c908c9..046f0605e2b7d 100644 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -131,7 +131,7 @@ func init() { Help: "Microsoft Cloud for US Government", }, { Value: regionDE, - Help: "Microsoft Cloud Germany", + Help: "Microsoft Cloud Germany (deprecated - try " + regionGlobal + " region first).", }, { Value: regionCN, Help: "Azure and Office 365 operated by Vnet Group in China", From 83e04ead37a31bc86a02c23caac5aa8034391e5e Mon Sep 17 00:00:00 2001 From: jkpe <16779171+jkpe@users.noreply.github.com> Date: Thu, 23 Jan 2025 12:33:14 +0000 Subject: [PATCH 15/73] s3: add DigitalOcean regions SFO2, LON1, TOR1, BLR1 --- backend/s3/s3.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index c26d63a3cb703..af149a306b993 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -1356,6 +1356,10 @@ func init() { Value: "sfo3.digitaloceanspaces.com", Help: "DigitalOcean Spaces San Francisco 3", Provider: "DigitalOcean", + }, { + Value: "sfo2.digitaloceanspaces.com", + Help: "DigitalOcean Spaces San Francisco 2", + Provider: "DigitalOcean", }, { Value: "fra1.digitaloceanspaces.com", Help: "DigitalOcean Spaces Frankfurt 1", @@ -1372,6 +1376,18 @@ func init() { Value: "sgp1.digitaloceanspaces.com", Help: "DigitalOcean Spaces Singapore 1", Provider: "DigitalOcean", + }, { + Value: "lon1.digitaloceanspaces.com", + Help: "DigitalOcean Spaces London 1", + Provider: "DigitalOcean", + }, { + Value: "tor1.digitaloceanspaces.com", + Help: "DigitalOcean Spaces Toronto 1", + Provider: "DigitalOcean", + }, { + Value: "blr1.digitaloceanspaces.com", + Help: "DigitalOcean Spaces Bangalore 1", + Provider: "DigitalOcean", }, { Value: "localhost:8333", Help: "SeaweedFS S3 localhost", From f79f929e57da2d339f4c7c2f8cea2ddab21e3908 Mon Sep 17 00:00:00 2001 From: Zachary Vorhies Date: Sat, 1 Feb 2025 05:20:09 -0700 Subject: [PATCH 16/73] serve nfs: update docs to note Windows is not supported - fixes #8352 --- docs/content/commands/rclone_serve_nfs.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/commands/rclone_serve_nfs.md b/docs/content/commands/rclone_serve_nfs.md index 72081cf56df34..ed8358ec24472 100644 --- a/docs/content/commands/rclone_serve_nfs.md +++ b/docs/content/commands/rclone_serve_nfs.md @@ -7,6 +7,8 @@ versionIntroduced: v1.65 --- # rclone serve nfs +*Not available in Windows.* + Serve the remote as an NFS mount ## Synopsis From 504f2fb571a0d9289d01c74616ee5373020d0b99 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sun, 2 Feb 2025 11:14:38 +0000 Subject: [PATCH 17/73] b2: fix "fatal error: concurrent map writes" - fixes #8355 This was caused by the embryonic metadata support. Since this isn't actually visible externally, this patch removes it for the time being. --- backend/b2/b2.go | 25 +++++++------------------ backend/b2/b2_internal_test.go | 6 ------ 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/backend/b2/b2.go b/backend/b2/b2.go index 4d4e754242566..0d1a4864c4ad9 100644 --- a/backend/b2/b2.go +++ b/backend/b2/b2.go @@ -299,14 +299,13 @@ type Fs struct { // Object describes a b2 object type Object struct { - fs *Fs // what this object is part of - remote string // The remote path - id string // b2 id of the file - modTime time.Time // The modified time of the object if known - sha1 string // SHA-1 hash if known - size int64 // Size of the object - mimeType string // Content-Type of the object - meta map[string]string // The object metadata if known - may be nil - with lower case keys + fs *Fs // what this object is part of + remote string // The remote path + id string // b2 id of the file + modTime time.Time // The modified time of the object if known + sha1 string // SHA-1 hash if known + size int64 // Size of the object + mimeType string // Content-Type of the object } // ------------------------------------------------------------ @@ -1598,9 +1597,6 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp if err != nil { return err } - // For now, just set "mtime" in metadata - o.meta = make(map[string]string, 1) - o.meta["mtime"] = o.modTime.Format(time.RFC3339Nano) return nil } @@ -1880,13 +1876,6 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open Info: Info, } - // Embryonic metadata support - just mtime - o.meta = make(map[string]string, 1) - modTime, err := parseTimeStringHelper(info.Info[timeKey]) - if err == nil { - o.meta["mtime"] = modTime.Format(time.RFC3339Nano) - } - // When reading files from B2 via cloudflare using // --b2-download-url cloudflare strips the Content-Length // headers (presumably so it can inject stuff) so use the old diff --git a/backend/b2/b2_internal_test.go b/backend/b2/b2_internal_test.go index e6bd8b404db1d..a6dccf5b53369 100644 --- a/backend/b2/b2_internal_test.go +++ b/backend/b2/b2_internal_test.go @@ -256,12 +256,6 @@ func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string assert.Equal(t, v, got, k) } - // mtime - for k, v := range metadata { - got := o.meta[k] - assert.Equal(t, v, got, k) - } - assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type") // Modification time from the x-bz-info-src_last_modified_millis header From 9dfce11c9babe56ac60f8361a26a44ccc7f8a18c Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 1 Feb 2025 12:07:25 +0000 Subject: [PATCH 18/73] lib/oauthutil: fix redirect URL mismatch errors - fixes #8351 In this commit we introduced support for client credentials flow: 65012beea4b93b7f lib/oauthutil: add support for OAuth client credential flow This involved re-organising the oauth credentials. Unfortunately a small error was made which used a fixed redirect URL rather than the one configured for the backend. This caused the box backend oauth flow not to work properly with redirect_uri_mismatch errors. These backends were using the wrong redirect URL and will likely be affected, though it is possible the backends have workarounds. - box - drive - googlecloudstorage - googlephotos - hidrive - pikpak - premiumizeme - sharefile - yandex --- lib/oauthutil/oauthutil.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index a03fd592d794c..e2cbc8837f511 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -108,7 +108,7 @@ func (conf *Config) MakeOauth2Config() *oauth2.Config { return &oauth2.Config{ ClientID: conf.ClientID, ClientSecret: conf.ClientSecret, - RedirectURL: RedirectLocalhostURL, + RedirectURL: conf.RedirectURL, Scopes: conf.Scopes, Endpoint: oauth2.Endpoint{ AuthURL: conf.AuthURL, From d953c0c51b668598bcc5731102b1ed469eb6ab3f Mon Sep 17 00:00:00 2001 From: nielash Date: Sun, 9 Feb 2025 21:13:55 -0500 Subject: [PATCH 19/73] bisync: fix listings missing concurrent modifications - fixes #8359 Before this change, there was a bug affecting listing files when: - a given bisync run had changes in the 2to1 direction AND - the run had NO changes in the 1to2 direction AND - at least one of the changed files changed AGAIN during the run (specifically, after the initial march and before the transfers.) In this situation, the listings on one side would still retain the prior version of the changed file, potentially causing conflicts or errors. This change fixes the issue by making sure that if we're updating the listings on one side, we must also update the other. (We previously tried to skip it for efficiency, but this failed to account for the possibility that a changed file could change again during the run.) --- cmd/bisync/bisync_test.go | 10 +++ cmd/bisync/deltas.go | 16 ++-- cmd/bisync/operations.go | 18 +---- ...testdir_path1.._testdir_path2.copy2to1.que | 1 + .../_testdir_path1.._testdir_path2.path1.lst | 10 +++ ...estdir_path1.._testdir_path2.path1.lst-new | 10 +++ ...estdir_path1.._testdir_path2.path1.lst-old | 10 +++ .../_testdir_path1.._testdir_path2.path2.lst | 10 +++ ...estdir_path1.._testdir_path2.path2.lst-new | 10 +++ ...estdir_path1.._testdir_path2.path2.lst-old | 10 +++ .../testdata/test_concurrent/golden/test.log | 73 +++++++++++++++++++ .../test_concurrent/initial/RCLONE_TEST | 1 + .../test_concurrent/initial/file1.txt | 0 .../test_concurrent/initial/file2.txt | 0 .../test_concurrent/initial/file3.txt | 0 .../test_concurrent/initial/file4.txt | 0 .../test_concurrent/initial/file5.txt | 0 .../test_concurrent/initial/file6.txt | 0 .../test_concurrent/initial/file7.txt | 0 .../test_concurrent/initial/file8.txt | 0 .../test_concurrent/modfiles/dummy.txt | 0 .../test_concurrent/modfiles/file1.txt | 1 + .../test_concurrent/modfiles/file10.txt | 1 + .../test_concurrent/modfiles/file11.txt | 1 + .../test_concurrent/modfiles/file2.txt | 1 + .../test_concurrent/modfiles/file5L.txt | 1 + .../test_concurrent/modfiles/file5R.txt | 1 + .../test_concurrent/modfiles/file6.txt | 1 + .../test_concurrent/modfiles/file7.txt | 1 + .../testdata/test_concurrent/scenario.txt | 15 ++++ docs/content/bisync.md | 3 + 31 files changed, 181 insertions(+), 24 deletions(-) create mode 100644 cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.copy2to1.que create mode 100644 cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst create mode 100644 cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-new create mode 100644 cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-old create mode 100644 cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst create mode 100644 cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-new create mode 100644 cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-old create mode 100644 cmd/bisync/testdata/test_concurrent/golden/test.log create mode 100644 cmd/bisync/testdata/test_concurrent/initial/RCLONE_TEST create mode 100644 cmd/bisync/testdata/test_concurrent/initial/file1.txt create mode 100644 cmd/bisync/testdata/test_concurrent/initial/file2.txt create mode 100644 cmd/bisync/testdata/test_concurrent/initial/file3.txt create mode 100644 cmd/bisync/testdata/test_concurrent/initial/file4.txt create mode 100644 cmd/bisync/testdata/test_concurrent/initial/file5.txt create mode 100644 cmd/bisync/testdata/test_concurrent/initial/file6.txt create mode 100644 cmd/bisync/testdata/test_concurrent/initial/file7.txt create mode 100644 cmd/bisync/testdata/test_concurrent/initial/file8.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/dummy.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/file1.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/file10.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/file11.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/file2.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/file5L.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/file5R.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/file6.txt create mode 100644 cmd/bisync/testdata/test_concurrent/modfiles/file7.txt create mode 100644 cmd/bisync/testdata/test_concurrent/scenario.txt diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go index da89c3e20e10c..f154d4d2871a2 100644 --- a/cmd/bisync/bisync_test.go +++ b/cmd/bisync/bisync_test.go @@ -746,6 +746,16 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) { case "test-func": b.TestFn = testFunc return + case "concurrent-func": + b.TestFn = func() { + src := filepath.Join(b.dataDir, "file7.txt") + dst := "file1.txt" + err := b.copyFile(ctx, src, b.replaceHex(b.path2), dst) + if err != nil { + fs.Errorf(src, "error copying file: %v", err) + } + } + return case "fix-names": // in case the local os converted any filenames ci.NoUnicodeNormalization = true diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index 1cb783d56ce3a..dde42e41413c3 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -286,7 +286,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string, } // applyDeltas -func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changes1, changes2 bool, results2to1, results1to2 []Results, queues queues, err error) { +func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (results2to1, results1to2 []Results, queues queues, err error) { path1 := bilib.FsPath(b.fs1) path2 := bilib.FsPath(b.fs2) @@ -367,7 +367,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change } } - //if there are potential conflicts to check, check them all here (outside the loop) in one fell swoop + // if there are potential conflicts to check, check them all here (outside the loop) in one fell swoop matches, err := b.checkconflicts(ctxCheck, filterCheck, b.fs1, b.fs2) for _, file := range ds1.sort() { @@ -392,7 +392,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change } else if d2.is(deltaOther) { b.indent("!WARNING", file, "New or changed in both paths") - //if files are identical, leave them alone instead of renaming + // if files are identical, leave them alone instead of renaming if (dirs1.has(file) || dirs1.has(alias)) && (dirs2.has(file) || dirs2.has(alias)) { fs.Infof(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file) ls1.getPut(file, skippedDirs1) @@ -486,7 +486,6 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change // Do the batch operation if copy2to1.NotEmpty() && !b.InGracefulShutdown { - changes1 = true b.indent("Path2", "Path1", "Do queued copies to") ctx = b.setBackupDir(ctx, 1) results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1") @@ -498,12 +497,11 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change return } - //copy empty dirs from path2 to path1 (if --create-empty-src-dirs) + // copy empty dirs from path2 to path1 (if --create-empty-src-dirs) b.syncEmptyDirs(ctx, b.fs1, copy2to1, dirs2, &results2to1, "make") } if copy1to2.NotEmpty() && !b.InGracefulShutdown { - changes2 = true b.indent("Path1", "Path2", "Do queued copies to") ctx = b.setBackupDir(ctx, 2) results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2") @@ -515,7 +513,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change return } - //copy empty dirs from path1 to path2 (if --create-empty-src-dirs) + // copy empty dirs from path1 to path2 (if --create-empty-src-dirs) b.syncEmptyDirs(ctx, b.fs2, copy1to2, dirs1, &results1to2, "make") } @@ -523,7 +521,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if err = b.saveQueue(delete1, "delete1"); err != nil { return } - //propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs) + // propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs) b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, &results2to1, "remove") } @@ -531,7 +529,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change if err = b.saveQueue(delete2, "delete2"); err != nil { return } - //propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs) + // propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs) b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, &results1to2, "remove") } diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go index 987a94e66f433..305680d745afb 100644 --- a/cmd/bisync/operations.go +++ b/cmd/bisync/operations.go @@ -359,8 +359,6 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) { // Determine and apply changes to Path1 and Path2 noChanges := ds1.empty() && ds2.empty() - changes1 := false // 2to1 - changes2 := false // 1to2 results2to1 := []Results{} results1to2 := []Results{} @@ -370,7 +368,7 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) { fs.Infof(nil, "No changes found") } else { fs.Infof(nil, "Applying changes") - changes1, changes2, results2to1, results1to2, queues, err = b.applyDeltas(octx, ds1, ds2) + results2to1, results1to2, queues, err = b.applyDeltas(octx, ds1, ds2) if err != nil { if b.InGracefulShutdown && (err == context.Canceled || err == accounting.ErrorMaxTransferLimitReachedGraceful || strings.Contains(err.Error(), "context canceled")) { fs.Infof(nil, "Ignoring sync error due to Graceful Shutdown: %v", err) @@ -395,21 +393,11 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) { } b.saveOldListings() // save new listings - // NOTE: "changes" in this case does not mean this run vs. last run, it means start of this run vs. end of this run. - // i.e. whether we can use the March lst-new as this side's lst without modifying it. if noChanges { b.replaceCurrentListings() } else { - if changes1 || b.InGracefulShutdown { // 2to1 - err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false) - } else { - err1 = bilib.CopyFileIfExists(b.newListing1, b.listing1) - } - if changes2 || b.InGracefulShutdown { // 1to2 - err2 = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true) - } else { - err2 = bilib.CopyFileIfExists(b.newListing2, b.listing2) - } + err1 = b.modifyListing(fctx, b.fs2, b.fs1, results2to1, queues, false) // 2to1 + err2 = b.modifyListing(fctx, b.fs1, b.fs2, results1to2, queues, true) // 1to2 } if b.DebugName != "" { l1, _ := b.loadListing(b.listing1) diff --git a/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.copy2to1.que new file mode 100644 index 0000000000000..029f3e190ed2c --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.copy2to1.que @@ -0,0 +1 @@ +"file1.txt" diff --git a/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst new file mode 100644 index 0000000000000..021dde2dc7589 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst @@ -0,0 +1,10 @@ +# bisync listing v1 from test +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt" diff --git a/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-new new file mode 100644 index 0000000000000..021dde2dc7589 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-new @@ -0,0 +1,10 @@ +# bisync listing v1 from test +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt" diff --git a/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-old new file mode 100644 index 0000000000000..021dde2dc7589 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path1.lst-old @@ -0,0 +1,10 @@ +# bisync listing v1 from test +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt" diff --git a/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst new file mode 100644 index 0000000000000..021dde2dc7589 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst @@ -0,0 +1,10 @@ +# bisync listing v1 from test +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt" diff --git a/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-new new file mode 100644 index 0000000000000..021dde2dc7589 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-new @@ -0,0 +1,10 @@ +# bisync listing v1 from test +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt" diff --git a/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-old new file mode 100644 index 0000000000000..021dde2dc7589 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/golden/_testdir_path1.._testdir_path2.path2.lst-old @@ -0,0 +1,10 @@ +# bisync listing v1 from test +- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST" +- 19 - - 2023-08-26T00:00:00.000000000+0000 "file1.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file2.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file3.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file4.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file5.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file6.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file7.txt" +- 0 - - 2000-01-01T00:00:00.000000000+0000 "file8.txt" diff --git a/cmd/bisync/testdata/test_concurrent/golden/test.log b/cmd/bisync/testdata/test_concurrent/golden/test.log new file mode 100644 index 0000000000000..35f894aeda860 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/golden/test.log @@ -0,0 +1,73 @@ +(01) : test concurrent + +(02) : test initial bisync +(03) : bisync resync +INFO : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set. +INFO : Bisyncing with Comparison Settings: +{ +"Modtime": true, +"Size": true, +"Checksum": false, +"NoSlowHash": false, +"SlowHashSyncOnly": false, +"DownloadHash": false +} +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Copying Path2 files to Path1 +INFO : - Path2 Resync is copying files to - Path1 +INFO : - Path1 Resync is copying files to - Path2 +INFO : Resync updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + +(04) : test changed on one path - file1 +(05) : touch-glob 2001-01-02 {datadir/} file5R.txt +(06) : touch-glob 2023-08-26 {datadir/} file7.txt +(07) : copy-as {datadir/}file5R.txt {path2/} file1.txt + +(08) : test bisync with file changed during +(09) : concurrent-func +(10) : bisync +INFO : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set. +INFO : Bisyncing with Comparison Settings: +{ +"Modtime": true, +"Size": true, +"Checksum": false, +"NoSlowHash": false, +"SlowHashSyncOnly": false, +"DownloadHash": false +} +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings +INFO : Path1 checking for diffs +INFO : Path2 checking for diffs +INFO : - Path2 File changed: size (larger), time (newer) - file1.txt +INFO : Path2: 1 changes:  0 new,  1 modified,  0 deleted +INFO : (Modified:  1 newer,  0 older,  1 larger,  0 smaller) +INFO : Applying changes +INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt +INFO : - Path2 Do queued copies to - Path1 +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful + +(11) : bisync +INFO : Setting --ignore-listing-checksum as neither --checksum nor --compare checksum are set. +INFO : Bisyncing with Comparison Settings: +{ +"Modtime": true, +"Size": true, +"Checksum": false, +"NoSlowHash": false, +"SlowHashSyncOnly": false, +"DownloadHash": false +} +INFO : Synching Path1 "{path1/}" with Path2 "{path2/}" +INFO : Building Path1 and Path2 listings +INFO : Path1 checking for diffs +INFO : Path2 checking for diffs +INFO : No changes found +INFO : Updating listings +INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}" +INFO : Bisync successful diff --git a/cmd/bisync/testdata/test_concurrent/initial/RCLONE_TEST b/cmd/bisync/testdata/test_concurrent/initial/RCLONE_TEST new file mode 100644 index 0000000000000..d8ca97c2a4dbe --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/initial/RCLONE_TEST @@ -0,0 +1 @@ +This file is used for testing the health of rclone accesses to the local/remote file system. Do not delete. diff --git a/cmd/bisync/testdata/test_concurrent/initial/file1.txt b/cmd/bisync/testdata/test_concurrent/initial/file1.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/initial/file2.txt b/cmd/bisync/testdata/test_concurrent/initial/file2.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/initial/file3.txt b/cmd/bisync/testdata/test_concurrent/initial/file3.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/initial/file4.txt b/cmd/bisync/testdata/test_concurrent/initial/file4.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/initial/file5.txt b/cmd/bisync/testdata/test_concurrent/initial/file5.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/initial/file6.txt b/cmd/bisync/testdata/test_concurrent/initial/file6.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/initial/file7.txt b/cmd/bisync/testdata/test_concurrent/initial/file7.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/initial/file8.txt b/cmd/bisync/testdata/test_concurrent/initial/file8.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/dummy.txt b/cmd/bisync/testdata/test_concurrent/modfiles/dummy.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/file1.txt b/cmd/bisync/testdata/test_concurrent/modfiles/file1.txt new file mode 100644 index 0000000000000..464147f09c0c8 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/modfiles/file1.txt @@ -0,0 +1 @@ +This file is newer diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/file10.txt b/cmd/bisync/testdata/test_concurrent/modfiles/file10.txt new file mode 100644 index 0000000000000..464147f09c0c8 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/modfiles/file10.txt @@ -0,0 +1 @@ +This file is newer diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/file11.txt b/cmd/bisync/testdata/test_concurrent/modfiles/file11.txt new file mode 100644 index 0000000000000..464147f09c0c8 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/modfiles/file11.txt @@ -0,0 +1 @@ +This file is newer diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/file2.txt b/cmd/bisync/testdata/test_concurrent/modfiles/file2.txt new file mode 100644 index 0000000000000..0fd70321acd6b --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/modfiles/file2.txt @@ -0,0 +1 @@ +Newer version \ No newline at end of file diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/file5L.txt b/cmd/bisync/testdata/test_concurrent/modfiles/file5L.txt new file mode 100644 index 0000000000000..43ceff1dbe273 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/modfiles/file5L.txt @@ -0,0 +1 @@ +This file is newer and not equal to 5R diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/file5R.txt b/cmd/bisync/testdata/test_concurrent/modfiles/file5R.txt new file mode 100644 index 0000000000000..a928fcf1382e0 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/modfiles/file5R.txt @@ -0,0 +1 @@ +This file is newer and not equal to 5L diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/file6.txt b/cmd/bisync/testdata/test_concurrent/modfiles/file6.txt new file mode 100644 index 0000000000000..464147f09c0c8 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/modfiles/file6.txt @@ -0,0 +1 @@ +This file is newer diff --git a/cmd/bisync/testdata/test_concurrent/modfiles/file7.txt b/cmd/bisync/testdata/test_concurrent/modfiles/file7.txt new file mode 100644 index 0000000000000..464147f09c0c8 --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/modfiles/file7.txt @@ -0,0 +1 @@ +This file is newer diff --git a/cmd/bisync/testdata/test_concurrent/scenario.txt b/cmd/bisync/testdata/test_concurrent/scenario.txt new file mode 100644 index 0000000000000..19ca9fcc2608f --- /dev/null +++ b/cmd/bisync/testdata/test_concurrent/scenario.txt @@ -0,0 +1,15 @@ +test concurrent + +test initial bisync +bisync resync + +test changed on one path - file1 +touch-glob 2001-01-02 {datadir/} file5R.txt +touch-glob 2023-08-26 {datadir/} file7.txt +copy-as {datadir/}file5R.txt {path2/} file1.txt + +test bisync with file changed during +concurrent-func +bisync + +bisync \ No newline at end of file diff --git a/docs/content/bisync.md b/docs/content/bisync.md index 458f92eab7246..3aa0bae540efb 100644 --- a/docs/content/bisync.md +++ b/docs/content/bisync.md @@ -1815,6 +1815,9 @@ about _Unison_ and synchronization in general. ## Changelog +### `v1.69.1` +* Fixed an issue causing listings to not capture concurrent modifications under certain conditions + ### `v1.68` * Fixed an issue affecting backends that round modtimes to a lower precision. From 9978750a8c00a905867231da2b3137102e81dbe3 Mon Sep 17 00:00:00 2001 From: nielash Date: Wed, 12 Feb 2025 00:57:52 -0500 Subject: [PATCH 20/73] fix golangci-lint errors --- backend/chunker/chunker.go | 2 +- backend/drive/drive.go | 35 ++++++++++++++++++++++------------ backend/sugarsync/sugarsync.go | 8 ++++---- cmd/bisync/bisync_test.go | 5 ++--- cmd/bisync/deltas.go | 4 +--- 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/backend/chunker/chunker.go b/backend/chunker/chunker.go index 7b1df95c1de26..3cc4b1c898716 100644 --- a/backend/chunker/chunker.go +++ b/backend/chunker/chunker.go @@ -2480,7 +2480,7 @@ func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte) if len(data) > maxMetadataSizeWritten { return nil, false, ErrMetaTooBig } - if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' { + if len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' { return nil, false, errors.New("invalid json") } var metadata metaSimpleJSON diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 8e878751aa4b9..4ef3b9befd90a 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -203,7 +203,6 @@ func driveScopesContainsAppFolder(scopes []string) bool { if scope == scopePrefix+"drive.appfolder" { return true } - } return false } @@ -1212,6 +1211,7 @@ func fixMimeType(mimeTypeIn string) string { } return mimeTypeOut } + func fixMimeTypeMap(in map[string][]string) (out map[string][]string) { out = make(map[string][]string, len(in)) for k, v := range in { @@ -1222,9 +1222,11 @@ func fixMimeTypeMap(in map[string][]string) (out map[string][]string) { } return out } + func isInternalMimeType(mimeType string) bool { return strings.HasPrefix(mimeType, "application/vnd.google-apps.") } + func isLinkMimeType(mimeType string) bool { return strings.HasPrefix(mimeType, "application/x-link-") } @@ -1657,7 +1659,8 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.F // When the drive.File cannot be represented as an fs.Object it will return (nil, nil). func (f *Fs) newObjectWithExportInfo( ctx context.Context, remote string, info *drive.File, - extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) { + extension, exportName, exportMimeType string, isDocument bool, +) (o fs.Object, err error) { // Note that resolveShortcut will have been called already if // we are being called from a listing. However the drive.Item // will have been resolved so this will do nothing. @@ -1848,6 +1851,7 @@ func linkTemplate(mt string) *template.Template { }) return _linkTemplates[mt] } + func (f *Fs) fetchFormats(ctx context.Context) { fetchFormatsOnce.Do(func() { var about *drive.About @@ -1893,7 +1897,8 @@ func (f *Fs) importFormats(ctx context.Context) map[string][]string { // Look through the exportExtensions and find the first format that can be // converted. If none found then return ("", "", false) func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string) ( - extension, mimeType string, isDocument bool) { + extension, mimeType string, isDocument bool, +) { exportMimeTypes, isDocument := f.exportFormats(ctx)[itemMimeType] if isDocument { for _, _extension := range f.exportExtensions { @@ -2689,7 +2694,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error { if shortcutID != "" { return f.delete(ctx, shortcutID, f.opt.UseTrash) } - var trashedFiles = false + trashedFiles := false if check { found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool { if !item.Trashed { @@ -2926,7 +2931,6 @@ func (f *Fs) CleanUp(ctx context.Context) error { err := f.svc.Files.EmptyTrash().Context(ctx).Do() return f.shouldRetry(ctx, err) }) - if err != nil { return err } @@ -3187,6 +3191,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT } }() } + func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (pageToken string, err error) { var startPageToken *drive.StartPageToken err = f.pacer.Call(func() (bool, error) { @@ -3990,14 +3995,13 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str case "query": if len(arg) == 1 { query := arg[0] - var results, err = f.query(ctx, query) + results, err := f.query(ctx, query) if err != nil { return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err) } return results, nil - } else { - return nil, errors.New("need a query argument") } + return nil, errors.New("need a query argument") case "rescue": dirID := "" _, delete := opt["delete"] @@ -4057,6 +4061,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) { } return "", hash.ErrUnsupported } + func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) { if t != hash.MD5 && t != hash.SHA1 && t != hash.SHA256 { return "", hash.ErrUnsupported @@ -4071,7 +4076,8 @@ func (o *baseObject) Size() int64 { // getRemoteInfoWithExport returns a drive.File and the export settings for the remote func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) ( - info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) { + info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error, +) { leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false) if err != nil { if err == fs.ErrorDirNotFound { @@ -4284,12 +4290,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read } return o.baseObject.open(ctx, o.url, options...) } + func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { // Update the size with what we are reading as it can change from // the HEAD in the listing to this GET. This stops rclone marking // the transfer as corrupted. var offset, end int64 = 0, -1 - var newOptions = options[:0] + newOptions := options[:0] for _, o := range options { // Note that Range requests don't work on Google docs: // https://developers.google.com/drive/v3/web/manage-downloads#partial_download @@ -4316,9 +4323,10 @@ func (o *documentObject) Open(ctx context.Context, options ...fs.OpenOption) (in } return } + func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) { var offset, limit int64 = 0, -1 - var data = o.content + data := o.content for _, option := range options { switch x := option.(type) { case *fs.SeekOption: @@ -4343,7 +4351,8 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io. } func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader, - src fs.ObjectInfo) (info *drive.File, err error) { + src fs.ObjectInfo, +) (info *drive.File, err error) { // Make the API request to upload metadata and file data. size := src.Size() if size >= 0 && size < int64(o.fs.opt.UploadCutoff) { @@ -4421,6 +4430,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return nil } + func (o *documentObject) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error { srcMimeType := fs.MimeType(ctx, src) importMimeType := "" @@ -4516,6 +4526,7 @@ func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err er func (o *documentObject) ext() string { return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:] } + func (o *linkObject) ext() string { return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:] } diff --git a/backend/sugarsync/sugarsync.go b/backend/sugarsync/sugarsync.go index 880776698bdf0..04dc31fd8c5e0 100644 --- a/backend/sugarsync/sugarsync.go +++ b/backend/sugarsync/sugarsync.go @@ -120,7 +120,7 @@ func init() { srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(rootURL) // FIXME // FIXME - //err = f.pacer.Call(func() (bool, error) { + // err = f.pacer.Call(func() (bool, error) { resp, err = srv.CallXML(context.Background(), &opts, &authRequest, nil) // return shouldRetry(ctx, resp, err) //}) @@ -327,7 +327,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, ID string) (info *api.File, func (f *Fs) getAuthToken(ctx context.Context) error { fs.Debugf(f, "Renewing token") - var authRequest = api.TokenAuthRequest{ + authRequest := api.TokenAuthRequest{ AccessKeyID: withDefault(f.opt.AccessKeyID, accessKeyID), PrivateAccessKey: withDefault(f.opt.PrivateAccessKey, obscure.MustReveal(encryptedPrivateAccessKey)), RefreshToken: f.opt.RefreshToken, @@ -509,7 +509,7 @@ func errorHandler(resp *http.Response) (err error) { return fmt.Errorf("error reading error out of body: %w", err) } match := findError.FindSubmatch(body) - if match == nil || len(match) < 2 || len(match[1]) == 0 { + if len(match) < 2 || len(match[1]) == 0 { return fmt.Errorf("HTTP error %v (%v) returned body: %q", resp.StatusCode, resp.Status, body) } return fmt.Errorf("HTTP error %v (%v): %s", resp.StatusCode, resp.Status, match[1]) @@ -552,7 +552,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { // FindLeaf finds a directory of name leaf in the folder with ID pathID func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) { - //fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf) + // fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf) // Find the leaf in pathID found, err = f.listAll(ctx, pathID, nil, func(item *api.Collection) bool { if strings.EqualFold(item.Name, leaf) { diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go index f154d4d2871a2..c97af894dd882 100644 --- a/cmd/bisync/bisync_test.go +++ b/cmd/bisync/bisync_test.go @@ -881,10 +881,9 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) { if !ok || err != nil { fs.Logf(remotePath, "Can't find expected file %s (was it renamed by the os?) %v", args[1], err) return - } else { - // include hash of filename to make unicode form differences easier to see in logs - fs.Debugf(remotePath, "verified file exists at correct path. filename hash: %s", stringToHash(leaf)) } + // include hash of filename to make unicode form differences easier to see in logs + fs.Debugf(remotePath, "verified file exists at correct path. filename hash: %s", stringToHash(leaf)) return default: return fmt.Errorf("unknown command: %q", args[0]) diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go index dde42e41413c3..2fba379c68bf6 100644 --- a/cmd/bisync/deltas.go +++ b/cmd/bisync/deltas.go @@ -161,9 +161,7 @@ func (b *bisyncRun) findDeltas(fctx context.Context, f fs.Fs, oldListing string, return } - if err == nil { - err = b.checkListing(now, newListing, "current "+msg) - } + err = b.checkListing(now, newListing, "current "+msg) if err != nil { return } From fb648e4774cc9291e7f873137867a425ceaaea60 Mon Sep 17 00:00:00 2001 From: jbagwell-akamai <113531113+jbagwell-akamai@users.noreply.github.com> Date: Thu, 13 Feb 2025 03:36:22 -0600 Subject: [PATCH 21/73] s3: add latest Linode Object Storage endpoints Added missing Linode Object Storage endpoints AMS, MAA, CGK, LON, LAX, MAD, MEL, MIA, OSA, GRU, SIN --- backend/s3/s3.go | 37 +++++++++++++++++++++++++++++++++++-- docs/content/s3.md | 44 +++++++++++++++++++++++++++++++++----------- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index af149a306b993..716b35ffdd225 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -934,34 +934,67 @@ func init() { Help: "The default endpoint\nIran", }}, }, { - // Linode endpoints: https://www.linode.com/docs/products/storage/object-storage/guides/urls/#cluster-url-s3-endpoint + // Linode endpoints: https://techdocs.akamai.com/cloud-computing/docs/object-storage-product-limits#supported-endpoint-types-by-region Name: "endpoint", Help: "Endpoint for Linode Object Storage API.", Provider: "Linode", Examples: []fs.OptionExample{{ + Value: "nl-ams-1.linodeobjects.com", + Help: "Amsterdam (Netherlands), nl-ams-1", + }, { Value: "us-southeast-1.linodeobjects.com", Help: "Atlanta, GA (USA), us-southeast-1", + }, { + Value: "in-maa-1.linodeobjects.com", + Help: "Chennai (India), in-maa-1", }, { Value: "us-ord-1.linodeobjects.com", Help: "Chicago, IL (USA), us-ord-1", }, { Value: "eu-central-1.linodeobjects.com", Help: "Frankfurt (Germany), eu-central-1", + }, { + Value: "id-cgk-1.linodeobjects.com", + Help: "Jakarta (Indonesia), id-cgk-1", + }, { + Value: "gb-lon-1.linodeobjects.com", + Help: "London 2 (Great Britain), gb-lon-1", + }, { + Value: "us-lax-1.linodeobjects.com", + Help: "Los Angeles, CA (USA), us-lax-1", + }, { + Value: "es-mad-1.linodeobjects.com", + Help: "Madrid (Spain), es-mad-1", + }, { + Value: "au-mel-1.linodeobjects.com", + Help: "Melbourne (Australia), au-mel-1", + }, { + Value: "us-mia-1.linodeobjects.com", + Help: "Miami, FL (USA), us-mia-1", }, { Value: "it-mil-1.linodeobjects.com", Help: "Milan (Italy), it-mil-1", }, { Value: "us-east-1.linodeobjects.com", Help: "Newark, NJ (USA), us-east-1", + }, { + Value: "jp-osa-1.linodeobjects.com", + Help: "Osaka (Japan), jp-osa-1", }, { Value: "fr-par-1.linodeobjects.com", Help: "Paris (France), fr-par-1", + }, { + Value: "br-gru-1.linodeobjects.com", + Help: "São Paulo (Brazil), br-gru-1", }, { Value: "us-sea-1.linodeobjects.com", Help: "Seattle, WA (USA), us-sea-1", }, { Value: "ap-south-1.linodeobjects.com", - Help: "Singapore ap-south-1", + Help: "Singapore, ap-south-1", + }, { + Value: "sg-sin-1.linodeobjects.com", + Help: "Singapore 2, sg-sin-1", }, { Value: "se-sto-1.linodeobjects.com", Help: "Stockholm (Sweden), se-sto-1", diff --git a/docs/content/s3.md b/docs/content/s3.md index a71549e94be0b..4d96cde940698 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -4845,27 +4845,49 @@ Option endpoint. Endpoint for Linode Object Storage API. Choose a number from below, or type in your own value. Press Enter to leave empty. - 1 / Atlanta, GA (USA), us-southeast-1 + 1 / Amsterdam (Netherlands), nl-ams-1 + \ (nl-ams-1.linodeobjects.com) + 2 / Atlanta, GA (USA), us-southeast-1 \ (us-southeast-1.linodeobjects.com) - 2 / Chicago, IL (USA), us-ord-1 + 3 / Chennai (India), in-maa-1 + \ (in-maa-1.linodeobjects.com) + 4 / Chicago, IL (USA), us-ord-1 \ (us-ord-1.linodeobjects.com) - 3 / Frankfurt (Germany), eu-central-1 + 5 / Frankfurt (Germany), eu-central-1 \ (eu-central-1.linodeobjects.com) - 4 / Milan (Italy), it-mil-1 + 6 / Jakarta (Indonesia), id-cgk-1 + \ (id-cgk-1.linodeobjects.com) + 7 / London 2 (Great Britain), gb-lon-1 + \ (gb-lon-1.linodeobjects.com) + 8 / Los Angeles, CA (USA), us-lax-1 + \ (us-lax-1.linodeobjects.com) + 9 / Madrid (Spain), es-mad-1 + \ (es-mad-1.linodeobjects.com) +10 / Melbourne (Australia), au-mel-1 + \ (au-mel-1.linodeobjects.com) +11 / Miami, FL (USA), us-mia-1 + \ (us-mia-1.linodeobjects.com) +12 / Milan (Italy), it-mil-1 \ (it-mil-1.linodeobjects.com) - 5 / Newark, NJ (USA), us-east-1 +13 / Newark, NJ (USA), us-east-1 \ (us-east-1.linodeobjects.com) - 6 / Paris (France), fr-par-1 +14 / Osaka (Japan), jp-osa-1 + \ (jp-osa-1.linodeobjects.com) +15 / Paris (France), fr-par-1 \ (fr-par-1.linodeobjects.com) - 7 / Seattle, WA (USA), us-sea-1 +16 / São Paulo (Brazil), br-gru-1 + \ (br-gru-1.linodeobjects.com) +17 / Seattle, WA (USA), us-sea-1 \ (us-sea-1.linodeobjects.com) - 8 / Singapore ap-south-1 +18 / Singapore, ap-south-1 \ (ap-south-1.linodeobjects.com) - 9 / Stockholm (Sweden), se-sto-1 +19 / Singapore 2, sg-sin-1 + \ (sg-sin-1.linodeobjects.com) +20 / Stockholm (Sweden), se-sto-1 \ (se-sto-1.linodeobjects.com) -10 / Washington, DC, (USA), us-iad-1 +21 / Washington, DC, (USA), us-iad-1 \ (us-iad-1.linodeobjects.com) -endpoint> 3 +endpoint> 5 Option acl. Canned ACL used when creating buckets and storing or copying objects. From 5fa85f66feb893aab8c4d5c07c9587c703de2bf3 Mon Sep 17 00:00:00 2001 From: Zachary Vorhies Date: Thu, 13 Feb 2025 04:41:37 -0700 Subject: [PATCH 22/73] doc: add note on concurrency of rclone purge --- cmd/purge/purge.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/purge/purge.go b/cmd/purge/purge.go index d8cb9095a1819..a86570eacd1e8 100644 --- a/cmd/purge/purge.go +++ b/cmd/purge/purge.go @@ -22,6 +22,9 @@ include/exclude filters - everything will be removed. Use the delete files. To delete empty directories only, use command [rmdir](/commands/rclone_rmdir/) or [rmdirs](/commands/rclone_rmdirs/). +The concurrency of this operation is controlled by the ` + "`--checkers`" + ` global flag. However, some backends will +implement this command directly, in which case ` + "`--checkers`" + ` will be ignored. + **Important**: Since this can cause data loss, test first with the ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag. `, From 259dbbab55101b60ac0ac65c2cd2c29a0cc410f2 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Mon, 10 Feb 2025 17:13:59 +0000 Subject: [PATCH 23/73] vfs: fix the cache failing to upload symlinks when --links was specified Before this change, if --vfs-cache-mode writes or above was set and --links was in use, when a symlink was saved then the VFS failed to upload it. This meant when the VFS was restarted the link wasn't there any more. This was caused by the local backend, which we use to manage the VFS cache, picking up the global --links flag. This patch makes sure that the internal instantations of the local backend in the VFS cache don't ever use the --links flag or the --local-links flag even if specified on the command line. Fixes #8367 --- vfs/vfscache/cache.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go index 9e228aacf9a5e..0ffa30c76ecb4 100644 --- a/vfs/vfscache/cache.go +++ b/vfs/vfscache/cache.go @@ -227,7 +227,10 @@ func (c *Cache) createItemDir(name string) (string, error) { // getBackend gets a backend for a cache root dir func getBackend(ctx context.Context, parentPath string, name string, relativeDirPath string) (fs.Fs, error) { - path := fmt.Sprintf(":local,encoding='%v':%s/%s/%s", encoder.OS, parentPath, name, relativeDirPath) + // Make sure we turn off the global links flag as it overrides the backend specific one + ctx, ci := fs.AddConfig(ctx) + ci.Links = false + path := fmt.Sprintf(":local,encoding='%v',links=false:%s/%s/%s", encoder.OS, parentPath, name, relativeDirPath) return fscache.Get(ctx, path) } From 68bbd8017d3f07f5eb729ecf98ed78b3e43e8c77 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 14 Feb 2025 12:29:32 +0000 Subject: [PATCH 24/73] docs: add FileLu as sponsors and tidy sponsor logos --- docs/content/sponsor.md | 3 ++- docs/layouts/shortcodes/sponsor.html | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/content/sponsor.md b/docs/content/sponsor.md index e2e049a71c912..f45e624d6426b 100644 --- a/docs/content/sponsor.md +++ b/docs/content/sponsor.md @@ -57,7 +57,8 @@ off donation. Thank you very much to our sponsors: {{< sponsor src="/img/logos/idrive_e2.svg" width="300" height="200" title="Visit our sponsor IDrive e2" link="https://www.idrive.com/e2/?refer=rclone">}} -{{< sponsor src="/img/logos/warp.svg" width="300" height="200" title="Visit our sponsor warp.dev" link="https://www.warp.dev/?utm_source=rclone&utm_medium=referral&utm_campaign=rclone_20231103">}} +{{< sponsor src="/img/logos/warp.svg" width="285" height="200" title="Visit our sponsor warp.dev" link="https://www.warp.dev/?utm_source=rclone&utm_medium=referral&utm_campaign=rclone_20231103">}} {{< sponsor src="/img/logos/sia.svg" width="200" height="200" title="Visit our sponsor sia" link="https://sia.tech">}} {{< sponsor src="/img/logos/route4me.svg" width="400" height="200" title="Visit our sponsor Route4Me" link="https://route4me.com/">}} {{< sponsor src="/img/logos/rcloneview.svg" width="300" height="200" title="Visit our sponsor RcloneView" link="https://rcloneview.com/">}} +{{< sponsor src="/img/logos/filelu-rclone.svg" width="330" height="200" title="Visit our sponsor FileLu" link="https://filelu.com/">}} diff --git a/docs/layouts/shortcodes/sponsor.html b/docs/layouts/shortcodes/sponsor.html index 43e76d2373c7d..bde5f8cb200cd 100644 --- a/docs/layouts/shortcodes/sponsor.html +++ b/docs/layouts/shortcodes/sponsor.html @@ -1,3 +1,3 @@ - + From 30c9bab35dadee9d9924f41ba31598baa3f26f5e Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:43:18 +0530 Subject: [PATCH 25/73] Added parallel docker builds and caching for go build in the container Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- .../build_publish_beta_docker_image.yml | 77 ----- .../workflows/build_publish_docker_image.yml | 294 ++++++++++++++++++ .../workflows/build_publish_docker_plugin.yml | 45 +++ .../build_publish_release_docker_image.yml | 89 ------ 4 files changed, 339 insertions(+), 166 deletions(-) delete mode 100644 .github/workflows/build_publish_beta_docker_image.yml create mode 100644 .github/workflows/build_publish_docker_image.yml create mode 100644 .github/workflows/build_publish_docker_plugin.yml delete mode 100644 .github/workflows/build_publish_release_docker_image.yml diff --git a/.github/workflows/build_publish_beta_docker_image.yml b/.github/workflows/build_publish_beta_docker_image.yml deleted file mode 100644 index fa838c6ff0e4e..0000000000000 --- a/.github/workflows/build_publish_beta_docker_image.yml +++ /dev/null @@ -1,77 +0,0 @@ -name: Docker beta build - -on: - push: - branches: - - master -jobs: - build: - if: github.repository == 'rclone/rclone' - runs-on: ubuntu-latest - name: Build image job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - # This is the user that triggered the Workflow. In this case, it will - # either be the user whom created the Release or manually triggered - # the workflow_dispatch. - username: ${{ github.actor }} - # `secrets.GITHUB_TOKEN` is a secret that's automatically generated by - # GitHub Actions at the start of a workflow run to identify the job. - # This is used to authenticate against GitHub Container Registry. - # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret - # for more detailed information. - password: ${{ secrets.GITHUB_TOKEN }} - - name: Show disk usage - shell: bash - run: | - df -h . - - name: Build and publish image - uses: docker/build-push-action@v6 - with: - file: Dockerfile - context: . - push: true # push the image to ghcr - tags: | - ghcr.io/rclone/rclone:beta - rclone/rclone:beta - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 - cache-from: type=gha, scope=${{ github.workflow }} - cache-to: type=gha, mode=max, scope=${{ github.workflow }} - provenance: false - # Eventually cache will need to be cleared if builds more frequent than once a week - # https://github.com/docker/build-push-action/issues/252 - - name: Show disk usage - shell: bash - run: | - df -h . diff --git a/.github/workflows/build_publish_docker_image.yml b/.github/workflows/build_publish_docker_image.yml new file mode 100644 index 0000000000000..d8e5f05579fa7 --- /dev/null +++ b/.github/workflows/build_publish_docker_image.yml @@ -0,0 +1,294 @@ +--- +# Github Actions release for rclone +# -*- compile-command: "yamllint -f build_publish_docker_image.yml" -*- + +name: Build & Push Docker Images + +# Trigger the workflow on push or pull request +on: + push: + branches: + - '**' + tags: + - '**' + pull_request: + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true + +jobs: + build-image: + if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runs-on: ubuntu-24.04 + - platform: linux/386 + runs-on: ubuntu-24.04 + - platform: linux/arm64 + runs-on: ubuntu-24.04-arm + - platform: linux/arm/v7 + runs-on: ubuntu-24.04-arm + - platform: linux/arm/v6 + runs-on: ubuntu-24.04-arm + + name: Build Docker Image for ${{ matrix.platform }} + runs-on: ${{ matrix.runs-on }} + + steps: + - name: Free Space + shell: bash + run: | + df -h . + # Remove android SDK + sudo rm -rf /usr/local/lib/android || true + # Remove .net runtime + sudo rm -rf /usr/share/dotnet || true + df -h . + + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set REPO_NAME Variable + run: | + echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV} + + - name: Set PLATFORM Variable + run: | + platform=${{ matrix.platform }} + echo "PLATFORM=${platform//\//-}" >> $GITHUB_ENV + + - name: Set CACHE_NAME Variable + shell: python + run: | + import os, re + + def slugify(input_string, max_length=63): + slug = input_string.lower() + slug = re.sub(r'[^a-z0-9 -]', ' ', slug) + slug = slug.strip() + slug = re.sub(r'\s+', '-', slug) + slug = re.sub(r'-+', '-', slug) + slug = slug[:max_length] + slug = re.sub(r'[-]+$', '', slug) + return slug + + ref_name_slug = "cache" + + if os.environ.get("GITHUB_REF_NAME") and os.environ['GITHUB_EVENT_NAME'] == "pull_request": + ref_name_slug += "-pr-" + slugify(os.environ['GITHUB_REF_NAME']) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"CACHE_NAME={ref_name_slug}\n") + + - name: Get ImageOS + # There's no way around this, because "ImageOS" is only available to + # processes, but the setup-go action uses it in its key. + id: imageos + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + return process.env.ImageOS + + - name: Extract Metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,manifest-descriptor # Important for digest annotation (used by Github packages) + with: + images: | + ghcr.io/${{ env.REPO_NAME }} + labels: | + org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone + org.opencontainers.image.vendor=${{ github.repository_owner }} + org.opencontainers.image.authors=rclone + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + tags: | + type=sha + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=beta,enable={{is_default_branch}} + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Load Go Build Cache for Docker + id: go-cache + uses: actions/cache@v4 + with: + key: ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }}-${{ hashFiles('**/go.mod') }}-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-${{ steps.imageos.outputs.result }}-go-${{ env.CACHE_NAME }}-${{ env.PLATFORM }} + # Cache only the go builds, the module download is cached via the docker layer caching + path: | + go-build-cache + + - name: Inject Go Build Cache into Docker + uses: reproducible-containers/buildkit-cache-dance@v3 + with: + cache-map: | + { + "go-build-cache": "/root/.cache/go-build" + } + skip-extraction: ${{ steps.go-cache.outputs.cache-hit }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + # This is the user that triggered the Workflow. In this case, it will + # either be the user whom created the Release or manually triggered + # the workflow_dispatch. + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Publish Image Digest + id: build + uses: docker/build-push-action@v6 + with: + file: Dockerfile + context: . + provenance: false + # don't specify 'tags' here (error "get can't push tagged ref by digest") + # tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + platforms: ${{ matrix.platform }} + outputs: | + type=image,name=ghcr.io/${{ env.REPO_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: | + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }} + cache-to: | + type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:build-${{ env.CACHE_NAME }}-${{ env.PLATFORM }},image-manifest=true,mode=max,compression=zstd + + - name: Export Image Digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload Image Digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM }} + path: /tmp/digests/* + retention-days: 1 + if-no-files-found: error + + merge-image: + runs-on: ubuntu-24.04 + needs: + - build-image + + steps: + - name: Download Image Digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set REPO_NAME Variable + run: | + echo "REPO_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >> ${GITHUB_ENV} + + - name: Extract Metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: index + with: + images: | + ${{ env.REPO_NAME }} + ghcr.io/${{ env.REPO_NAME }} + labels: | + org.opencontainers.image.url=https://github.com/rclone/rclone/pkgs/container/rclone + org.opencontainers.image.vendor=${{ github.repository_owner }} + org.opencontainers.image.authors=rclone + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + tags: | + type=sha + type=ref,event=pr + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=beta,enable={{is_default_branch}} + + - name: Extract Tags + shell: python + run: | + import json, os + + metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON'] + metadata = json.loads(metadata_json) + + tags = [f"--tag '{tag}'" for tag in metadata["tags"]] + tags_string = " ".join(tags) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"TAGS={tags_string}\n") + + - name: Extract Annotations + shell: python + run: | + import json, os + + metadata_json = os.environ['DOCKER_METADATA_OUTPUT_JSON'] + metadata = json.loads(metadata_json) + + annotations = [f"--annotation '{annotation}'" for annotation in metadata["annotations"]] + annotations_string = " ".join(annotations) + + with open(os.environ['GITHUB_ENV'], 'a') as env: + env.write(f"ANNOTATIONS={annotations_string}\n") + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + # This is the user that triggered the Workflow. In this case, it will + # either be the user whom created the Release or manually triggered + # the workflow_dispatch. + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create & Push Manifest List + working-directory: /tmp/digests + run: | + docker buildx imagetools create \ + ${{ env.TAGS }} \ + ${{ env.ANNOTATIONS }} \ + $(printf 'ghcr.io/${{ env.REPO_NAME }}@sha256:%s ' *) + + - name: Inspect and Run Multi-Platform Image + run: | + docker buildx imagetools inspect --raw ${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} + docker buildx imagetools inspect --raw ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} + docker run --rm ghcr.io/${{ env.REPO_NAME }}:${{ steps.meta.outputs.version }} version diff --git a/.github/workflows/build_publish_docker_plugin.yml b/.github/workflows/build_publish_docker_plugin.yml new file mode 100644 index 0000000000000..4056cb04be316 --- /dev/null +++ b/.github/workflows/build_publish_docker_plugin.yml @@ -0,0 +1,45 @@ +--- +# Github Actions release for rclone +# -*- compile-command: "yamllint -f parsable build_publish_docker_plugin.yml" -*- + +name: Release Build for Docker Plugin + +on: + release: + types: [published] + +jobs: + + build_docker_volume_plugin: + if: github.repository == 'rclone/rclone' + needs: build + runs-on: ubuntu-latest + name: Build docker plugin job + steps: + - name: Free some space + shell: bash + run: | + df -h . + # Remove android SDK + sudo rm -rf /usr/local/lib/android || true + # Remove .net runtime + sudo rm -rf /usr/share/dotnet || true + df -h . + - name: Checkout master + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build and publish docker plugin + shell: bash + run: | + VER=${GITHUB_REF#refs/tags/} + PLUGIN_USER=rclone + docker login --username ${{ secrets.DOCKER_HUB_USER }} \ + --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" + for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do + export PLUGIN_USER PLUGIN_ARCH + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} + done + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} diff --git a/.github/workflows/build_publish_release_docker_image.yml b/.github/workflows/build_publish_release_docker_image.yml deleted file mode 100644 index b69285a7e5bb8..0000000000000 --- a/.github/workflows/build_publish_release_docker_image.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Docker release build - -on: - release: - types: [published] - -jobs: - build: - if: github.repository == 'rclone/rclone' - runs-on: ubuntu-latest - name: Build image job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Get actual patch version - id: actual_patch_version - run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g') - - name: Get actual minor version - id: actual_minor_version - run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2) - - name: Get actual major version - id: actual_major_version - run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1) - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Build and publish image - uses: docker/build-push-action@v6 - with: - file: Dockerfile - context: . - platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 - push: true - tags: | - rclone/rclone:latest - rclone/rclone:${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }} - rclone/rclone:${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }} - rclone/rclone:${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }} - - build_docker_volume_plugin: - if: github.repository == 'rclone/rclone' - needs: build - runs-on: ubuntu-latest - name: Build docker plugin job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Build and publish docker plugin - shell: bash - run: | - VER=${GITHUB_REF#refs/tags/} - PLUGIN_USER=rclone - docker login --username ${{ secrets.DOCKER_HUB_USER }} \ - --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" - for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do - export PLUGIN_USER PLUGIN_ARCH - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} - done - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} From b63c42f39bd5ab9fe9385ad2274efc34b0b272ee Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Tue, 11 Feb 2025 19:08:46 +0530 Subject: [PATCH 26/73] build: disable docker builds on PRs & add missing dockerfile changes Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- .../workflows/build_publish_docker_image.yml | 6 +-- Dockerfile | 44 +++++++++++++++---- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_publish_docker_image.yml b/.github/workflows/build_publish_docker_image.yml index d8e5f05579fa7..abc017bc347ac 100644 --- a/.github/workflows/build_publish_docker_image.yml +++ b/.github/workflows/build_publish_docker_image.yml @@ -1,6 +1,6 @@ --- # Github Actions release for rclone -# -*- compile-command: "yamllint -f build_publish_docker_image.yml" -*- +# -*- compile-command: "yamllint -f parsable build_publish_docker_image.yml" -*- name: Build & Push Docker Images @@ -11,7 +11,6 @@ on: - '**' tags: - '**' - pull_request: workflow_dispatch: inputs: manual: @@ -21,7 +20,7 @@ on: jobs: build-image: - if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) + if: inputs.manual || (github.repository == 'rclone/rclone' && github.event_name != 'pull_request') timeout-minutes: 60 strategy: fail-fast: false @@ -192,6 +191,7 @@ jobs: if-no-files-found: error merge-image: + name: Merge & Push Final Docker Image runs-on: ubuntu-24.04 needs: - build-image diff --git a/Dockerfile b/Dockerfile index 82f16aad784a1..bb086159492c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,47 @@ FROM golang:alpine AS builder -COPY . /go/src/github.com/rclone/rclone/ +ARG CGO_ENABLED=0 + WORKDIR /go/src/github.com/rclone/rclone/ -RUN apk add --no-cache make bash gawk git -RUN \ - CGO_ENABLED=0 \ - make -RUN ./rclone version +RUN echo "**** Set Go Environment Variables ****" && \ + go env -w GOCACHE=/root/.cache/go-build + +RUN echo "**** Install Dependencies ****" && \ + apk add --no-cache \ + make \ + bash \ + gawk \ + git + +COPY go.mod . +COPY go.sum . + +RUN echo "**** Download Go Dependencies ****" && \ + go mod download -x + +RUN echo "**** Verify Go Dependencies ****" && \ + go mod verify + +COPY . . + +RUN --mount=type=cache,target=/root/.cache/go-build,sharing=locked \ + echo "**** Build Binary ****" && \ + make + +RUN echo "**** Print Version Binary ****" && \ + ./rclone version # Begin final image FROM alpine:latest -RUN apk --no-cache add ca-certificates fuse3 tzdata && \ - echo "user_allow_other" >> /etc/fuse.conf +RUN echo "**** Install Dependencies ****" && \ + apk add --no-cache \ + ca-certificates \ + fuse3 \ + tzdata && \ + echo "Enable user_allow_other in fuse" && \ + echo "user_allow_other" >> /etc/fuse.conf COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/ From 4e77a4ff73645f45873188de2683067d6756038e Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 14 Feb 2025 15:17:21 +0000 Subject: [PATCH 27/73] Version v1.69.1 --- MANUAL.html | 644 +++++++++++------- MANUAL.md | 230 ++++++- MANUAL.txt | 253 ++++++- docs/content/changelog.md | 26 + docs/content/commands/rclone.md | 2 +- .../commands/rclone_config_encryption_set.md | 4 +- docs/content/commands/rclone_copyurl.md | 2 +- docs/content/commands/rclone_purge.md | 3 + docs/content/commands/rclone_serve_nfs.md | 4 +- docs/content/commands/rclone_serve_s3.md | 2 +- docs/content/flags.md | 4 +- docs/content/googlephotos.md | 2 +- docs/content/onedrive.md | 2 +- rclone.1 | 294 +++++++- 14 files changed, 1119 insertions(+), 353 deletions(-) diff --git a/MANUAL.html b/MANUAL.html index 29c5bfac1b4d4..0cbfcecd64d32 100644 --- a/MANUAL.html +++ b/MANUAL.html @@ -81,8 +81,74 @@

rclone(1) User Manual

Nick Craig-Wood

-

Jan 12, 2025

+

Feb 14, 2025

+

NAME

+

rclone - manage files on cloud storage

+

SYNOPSIS

+
Usage:
+  rclone [flags]
+  rclone [command]
+
+Available commands:
+  about       Get quota information from the remote.
+  authorize   Remote authorization.
+  backend     Run a backend-specific command.
+  bisync      Perform bidirectional synchronization between two paths.
+  cat         Concatenates any files and sends them to stdout.
+  check       Checks the files in the source and destination match.
+  checksum    Checks the files in the destination against a SUM file.
+  cleanup     Clean up the remote if possible.
+  completion  Output completion script for a given shell.
+  config      Enter an interactive configuration session.
+  copy        Copy files from source to dest, skipping identical files.
+  copyto      Copy files from source to dest, skipping identical files.
+  copyurl     Copy the contents of the URL supplied content to dest:path.
+  cryptcheck  Cryptcheck checks the integrity of an encrypted remote.
+  cryptdecode Cryptdecode returns unencrypted file names.
+  dedupe      Interactively find duplicate filenames and delete/rename them.
+  delete      Remove the files in path.
+  deletefile  Remove a single file from remote.
+  gendocs     Output markdown docs for rclone to the directory supplied.
+  gitannex    Speaks with git-annex over stdin/stdout.
+  hashsum     Produces a hashsum file for all the objects in the path.
+  help        Show help for rclone commands, flags and backends.
+  link        Generate public link to file/folder.
+  listremotes List all the remotes in the config file and defined in environment variables.
+  ls          List the objects in the path with size and path.
+  lsd         List all directories/containers/buckets in the path.
+  lsf         List directories and objects in remote:path formatted for parsing.
+  lsjson      List directories and objects in the path in JSON format.
+  lsl         List the objects in path with modification time, size and path.
+  md5sum      Produces an md5sum file for all the objects in the path.
+  mkdir       Make the path if it doesn't already exist.
+  mount       Mount the remote as file system on a mountpoint.
+  move        Move files from source to dest.
+  moveto      Move file or directory from source to dest.
+  ncdu        Explore a remote with a text based user interface.
+  nfsmount    Mount the remote as file system on a mountpoint.
+  obscure     Obscure password for use in the rclone config file.
+  purge       Remove the path and all of its contents.
+  rc          Run a command against a running rclone.
+  rcat        Copies standard input to file on remote.
+  rcd         Run rclone listening to remote control commands only.
+  rmdir       Remove the empty directory at path.
+  rmdirs      Remove empty directories under the path.
+  selfupdate  Update the rclone binary.
+  serve       Serve a remote over a protocol.
+  settier     Changes storage class/tier of objects in remote.
+  sha1sum     Produces an sha1sum file for all the objects in the path.
+  size        Prints the total size and number of objects in remote:path.
+  sync        Make source and dest identical, modifying destination only.
+  test        Run a test command
+  touch       Create new file or change file modification time.
+  tree        List the contents of the remote in a tree like fashion.
+  version     Show the version number.
+
+Use "rclone [command] --help" for more information about a command.
+Use "rclone help flags" for to see the global flags.
+Use "rclone help backends" for a list of supported services.
+

Rclone syncs your files to cloud storage

rclone logo

    @@ -411,7 +477,7 @@

    Docker installation

    Snap installation

    Get it from the Snap Store

    Make sure you have Snapd installed

    -
    $ sudo snap install rclone
    +
    $ sudo snap install rclone

    Due to the strict confinement of Snap, rclone snap cannot access real /home/$USER/.config/rclone directory, default config path is as below.

    • Default config directory: @@ -585,7 +651,7 @@

      Subcommands

      rclone sync --interactive /local/path remote:path # syncs /local/path to the remote

      rclone config

      Enter an interactive configuration session.

      -

      Synopsis

      +

      Synopsis

      Enter an interactive configuration session where you can setup new remotes and manage existing ones. You may also set or remove a password to protect your configuration.

      rclone config [flags]

      Options

      @@ -613,7 +679,7 @@

      See Also

    rclone copy

    Copy files from source to dest, skipping identical files.

    -

    Synopsis

    +

    Synopsis

    Copy the source to the destination. Does not transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Doesn't delete files from the destination. If you want to also delete files from destination, to make it match source, use the sync command instead.

    Note that it is always the contents of the directory that is synced, not the directory itself. So when source:path is a directory, it's the contents of source:path that are copied, not the directory name and contents.

    To copy single files, use the copyto command instead.

    @@ -716,7 +782,7 @@

    See Also

rclone sync

Make source and dest identical, modifying destination only.

-

Synopsis

+

Synopsis

Sync the source to the destination, changing the destination only. Doesn't transfer files that are identical on source and destination, testing by size and modification time or MD5SUM. Destination is updated to match source, including deleting files if necessary (except duplicate objects, see below). If you don't want to delete files from destination, use the copy command instead.

Important: Since this can cause data loss, test first with the --dry-run or the --interactive/-i flag.

rclone sync --interactive SOURCE remote:DESTINATION
@@ -857,7 +923,7 @@

See Also

rclone move

Move files from source to dest.

-

Synopsis

+

Synopsis

Moves the contents of the source directory to the destination directory. Rclone will error if the source and destination overlap and the remote does not support a server-side directory move operation.

To move single files, use the moveto command instead.

If no filters are in use and if possible this will server-side move source:path into dest:path. After this source:path will no longer exist.

@@ -948,7 +1014,7 @@

See Also

rclone delete

Remove the files in path.

-

Synopsis

+

Synopsis

Remove the files in path. Unlike purge it obeys include/exclude filters so can be used to selectively delete files.

rclone delete only deletes files but leaves the directory structure alone. If you want to delete a directory and all of its contents use the purge command.

If you supply the --rmdirs flag, it will remove all empty directories along with it. You can also use the separate command rmdir or rmdirs to delete empty directories only.

@@ -1003,8 +1069,9 @@

See Also

rclone purge

Remove the path and all of its contents.

-

Synopsis

+

Synopsis

Remove the path and all of its contents. Note that this does not obey include/exclude filters - everything will be removed. Use the delete command if you want to selectively delete files. To delete empty directories only, use command rmdir or rmdirs.

+

The concurrency of this operation is controlled by the --checkers global flag. However, some backends will implement this command directly, in which case --checkers will be ignored.

Important: Since this can cause data loss, test first with the --dry-run or the --interactive/-i flag.

rclone purge remote:path [flags]

Options

@@ -1036,7 +1103,7 @@

See Also

rclone rmdir

Remove the empty directory at path.

-

Synopsis

+

Synopsis

This removes empty directory given by path. Will not remove the path if it has any objects in it, not even empty subdirectories. Use command rmdirs (or delete with option --rmdirs) to do that.

To delete a path and any objects in it, use purge command.

rclone rmdir remote:path [flags]
@@ -1054,7 +1121,7 @@

See Also

rclone check

Checks the files in the source and destination match.

-

Synopsis

+

Synopsis

Checks the files in the source and destination match. It compares sizes and hashes (MD5 or SHA1) and logs a report of files that don't match. It doesn't alter the source or destination.

For the crypt remote there is a dedicated command, cryptcheck, that are able to check the checksums of the encrypted files.

If you supply the --size-only flag, it will only compare the sizes not the hashes as well. Use this for a quick check.

@@ -1121,7 +1188,7 @@

See Also

rclone ls

List the objects in the path with size and path.

-

Synopsis

+

Synopsis

Lists the objects in the source path to standard output in a human readable format with size and path. Recurses by default.

Eg

$ rclone ls swift:bucket
@@ -1180,7 +1247,7 @@ 

See Also

rclone lsd

List all directories/containers/buckets in the path.

-

Synopsis

+

Synopsis

Lists the directories in the source path to standard output. Does not recurse by default. Use the -R flag to recurse.

This command lists the total size of the directory (if known, -1 if not), the modification time (if known, the current time if not), the number of objects in the directory (if known, -1 if not) and the name of the directory, Eg

$ rclone lsd swift:
@@ -1244,7 +1311,7 @@ 

See Also

rclone lsl

List the objects in path with modification time, size and path.

-

Synopsis

+

Synopsis

Lists the objects in the source path to standard output in a human readable format with modification time, size and path. Recurses by default.

Eg

$ rclone lsl swift:bucket
@@ -1303,7 +1370,7 @@ 

See Also

rclone md5sum

Produces an md5sum file for all the objects in the path.

-

Synopsis

+

Synopsis

Produces an md5sum file for all the objects in the path. This is in the same format as the standard md5sum tool produces.

By default, the hash is requested from the remote. If MD5 is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling MD5 for any remote.

For other algorithms, see the hashsum command. Running rclone md5sum remote:path is equivalent to running rclone hashsum MD5 remote:path.

@@ -1350,7 +1417,7 @@

See Also

rclone sha1sum

Produces an sha1sum file for all the objects in the path.

-

Synopsis

+

Synopsis

Produces an sha1sum file for all the objects in the path. This is in the same format as the standard sha1sum tool produces.

By default, the hash is requested from the remote. If SHA-1 is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling SHA-1 for any remote.

For other algorithms, see the hashsum command. Running rclone sha1sum remote:path is equivalent to running rclone hashsum SHA1 remote:path.

@@ -1398,7 +1465,7 @@

See Also

rclone size

Prints the total size and number of objects in remote:path.

-

Synopsis

+

Synopsis

Counts objects in the path and calculates the total size. Prints the result to standard output.

By default the output is in human-readable format, but shows values in both human-readable format as well as the raw numbers (global option --human-readable is not considered). Use option --json to format output as JSON instead.

Recurses by default, use --max-depth 1 to stop the recursion.

@@ -1442,7 +1509,7 @@

See Also

rclone version

Show the version number.

-

Synopsis

+

Synopsis

Show the rclone version number, the go version, the build target OS and architecture, the runtime OS and kernel version and bitness, build tags and the type of executable (static or dynamic).

For example:

$ rclone version
@@ -1478,7 +1545,7 @@ 

See Also

rclone cleanup

Clean up the remote if possible.

-

Synopsis

+

Synopsis

Clean up the remote if possible. Empty the trash or delete old file versions. Not supported by all remotes.

rclone cleanup remote:path [flags]

Options

@@ -1495,7 +1562,7 @@

See Also

rclone dedupe

Interactively find duplicate filenames and delete/rename them.

-

Synopsis

+

Synopsis

By default dedupe interactively finds files with duplicate names and offers to delete all but one or rename them to be different. This is known as deduping by name.

Deduping by name is only useful with a small group of backends (e.g. Google Drive, Opendrive) that can have duplicate file names. It can be run on wrapping backends (e.g. crypt) if they wrap a backend which supports duplicate file names.

However if --by-hash is passed in then dedupe will find files with duplicate hashes instead which will work on any backend which supports at least one hash. This can be used to find files with duplicate content. This is known as deduping by hash.

@@ -1579,7 +1646,7 @@

See Also

rclone about

Get quota information from the remote.

-

Synopsis

+

Synopsis

Prints quota information about a remote to standard output. The output is typically used, free, quota and trash contents.

E.g. Typical output from rclone about remote: is:

Total:   17 GiB
@@ -1625,7 +1692,7 @@ 

See Also

rclone authorize

Remote authorization.

-

Synopsis

+

Synopsis

Remote authorization. Used to authorize a remote or headless rclone from a machine with a browser - use as instructed by rclone config.

Use --auth-no-open-browser to prevent rclone to open auth link in default browser automatically.

Use --template to generate HTML output via a custom Go template. If a blank string is provided as an argument to this flag, the default template is used.

@@ -1641,7 +1708,7 @@

See Also

rclone backend

Run a backend-specific command.

-

Synopsis

+

Synopsis

This runs a backend-specific command. The commands themselves (except for "help" and "features") are defined by the backends and you should see the backend docs for definitions.

You can discover what commands a backend implements by using

rclone backend help remote:
@@ -1670,7 +1737,7 @@ 

See Also

rclone bisync

Perform bidirectional synchronization between two paths.

-

Synopsis

+

Synopsis

Perform bidirectional synchronization between two paths.

Bisync provides a bidirectional cloud sync solution in rclone. It retains the Path1 and Path2 filesystem listings from the prior run. On each successive run it will: - list files on Path1 and Path2, and check for changes on each side. Changes include New, Newer, Older, and Deleted files. - Propagate changes on Path1 to Path2, and vice-versa.

Bisync is in beta and is considered an advanced command, so use with care. Make sure you have read and understood the entire manual (especially the Limitations section) before using, or data loss can result. Questions can be asked in the Rclone Forum.

@@ -1773,7 +1840,7 @@

See Also

rclone cat

Concatenates any files and sends them to stdout.

-

Synopsis

+

Synopsis

Sends any files to standard output.

You can use it like this to output a single file

rclone cat remote:path/to/file
@@ -1833,7 +1900,7 @@

See Also

rclone checksum

Checks the files in the destination against a SUM file.

-

Synopsis

+

Synopsis

Checks that hashsums of destination files match the SUM file. It compares hashes (MD5, SHA1, etc) and logs a report of files which don't match. It doesn't alter the file system.

The sumfile is treated as the source and the dst:path is treated as the destination for the purposes of the output.

If you supply the --download flag, it will download the data from the remote and calculate the content hash on the fly. This can be useful for remotes that don't support hashes or if you really want to check all the data.

@@ -1895,7 +1962,7 @@

See Also

rclone completion

Output completion script for a given shell.

-

Synopsis

+

Synopsis

Generates a shell completion script for rclone. Run with --help to list the supported shells.

Options

  -h, --help   help for completion
@@ -1910,7 +1977,7 @@

See Also

rclone completion bash

Output bash completion script for rclone.

-

Synopsis

+

Synopsis

Generates a bash shell autocompletion script for rclone.

By default, when run without any arguments,

rclone completion bash
@@ -1933,7 +2000,7 @@

See Also

rclone completion fish

Output fish completion script for rclone.

-

Synopsis

+

Synopsis

Generates a fish autocompletion script for rclone.

This writes to /etc/fish/completions/rclone.fish by default so will probably need to be run with sudo or as root, e.g.

sudo rclone completion fish
@@ -1951,7 +2018,7 @@

See Also

rclone completion powershell

Output powershell completion script for rclone.

-

Synopsis

+

Synopsis

Generate the autocompletion script for powershell.

To load completions in your current shell session:

rclone completion powershell | Out-String | Invoke-Expression
@@ -1967,7 +2034,7 @@

See Also

rclone completion zsh

Output zsh completion script for rclone.

-

Synopsis

+

Synopsis

Generates a zsh autocompletion script for rclone.

This writes to /usr/share/zsh/vendor-completions/_rclone by default so will probably need to be run with sudo or as root, e.g.

sudo rclone completion zsh
@@ -1985,7 +2052,7 @@

See Also

rclone config create

Create a new remote with name, type and options.

-

Synopsis

+

Synopsis

Create a new remote of name with type and options. The options should be passed in pairs of key value or as key=value.

For example, to make a swift remote of name myremote using auto config you would do:

rclone config create myremote swift env_auth true
@@ -2066,7 +2133,7 @@ 

See Also

rclone config disconnect

Disconnects user from remote

-

Synopsis

+

Synopsis

This disconnects the remote: passed in to the cloud storage system.

This normally means revoking the oauth token.

To reconnect use "rclone config reconnect".

@@ -2090,7 +2157,7 @@

See Also

rclone config edit

Enter an interactive configuration session.

-

Synopsis

+

Synopsis

Enter an interactive configuration session where you can setup new remotes and manage existing ones. You may also set or remove a password to protect your configuration.

rclone config edit [flags]

Options

@@ -2102,7 +2169,7 @@

See Also

rclone config encryption

set, remove and check the encryption for the config file

-

Synopsis

+

Synopsis

This command sets, clears and checks the encryption for the config file using the subcommands below.

Options

  -h, --help   help for encryption
@@ -2116,7 +2183,7 @@

See Also

rclone config encryption check

Check that the config file is encrypted

-

Synopsis

+

Synopsis

This checks the config file is encrypted and that you can decrypt it.

It will attempt to decrypt the config using the password you supply.

If decryption fails it will return a non-zero exit code if using --password-command, otherwise it will prompt again for the password.

@@ -2131,7 +2198,7 @@

See Also

rclone config encryption remove

Remove the config file encryption password

-

Synopsis

+

Synopsis

Remove the config file encryption password

This removes the config file encryption, returning it to un-encrypted.

If --password-command is in use, this will be called to supply the old config password.

@@ -2146,12 +2213,12 @@

See Also

rclone config encryption set

Set or change the config file encryption password

-

Synopsis

+

Synopsis

This command sets or changes the config file encryption password.

If there was no config password set then it sets a new one, otherwise it changes the existing config password.

Note that if you are changing an encryption password using --password-command then this will be called once to decrypt the config using the old password and then again to read the new password to re-encrypt the config.

-

When --password-command is called to change the password then the environment variable RCLONE_PASSWORD_CHANGE=1 will be set. So if changing passwords programatically you can use the environment variable to distinguish which password you must supply.

-

Alternatively you can remove the password first (with rclone config encryption remove), then set it again with this command which may be easier if you don't mind the unecrypted config file being on the disk briefly.

+

When --password-command is called to change the password then the environment variable RCLONE_PASSWORD_CHANGE=1 will be set. So if changing passwords programmatically you can use the environment variable to distinguish which password you must supply.

+

Alternatively you can remove the password first (with rclone config encryption remove), then set it again with this command which may be easier if you don't mind the unencrypted config file being on the disk briefly.

rclone config encryption set [flags]

Options

  -h, --help   help for set
@@ -2172,7 +2239,7 @@

See Also

rclone config password

Update password in an existing remote.

-

Synopsis

+

Synopsis

Update an existing remote's password. The password should be passed in pairs of key password or as key=password. The password should be passed in in clear (unobscured).

For example, to set password of a remote of name myremote you would do:

rclone config password myremote fieldname mypassword
@@ -2208,7 +2275,7 @@ 

See Also

rclone config reconnect

Re-authenticates user with remote.

-

Synopsis

+

Synopsis

This reconnects remote: passed in to the cloud storage system.

To disconnect the remote use "rclone config disconnect".

This normally means going through the interactive oauth flow again.

@@ -2222,7 +2289,7 @@

See Also

rclone config redacted

Print redacted (decrypted) config file, or the redacted config for a single remote.

-

Synopsis

+

Synopsis

This prints a redacted copy of the config file, either the whole config file or for a given remote.

The config file will be redacted by replacing all passwords and other sensitive info with XXX.

This makes the config file suitable for posting online for support.

@@ -2257,7 +2324,7 @@

See Also

rclone config update

Update options in an existing remote.

-

Synopsis

+

Synopsis

Update an existing remote's options. The options should be passed in pairs of key value or as key=value.

For example, to update the env_auth field of a remote of name myremote you would do:

rclone config update myremote env_auth true
@@ -2328,7 +2395,7 @@ 

See Also

rclone config userinfo

Prints info about logged in user of remote.

-

Synopsis

+

Synopsis

This prints the details of the person logged in to the cloud storage system.

rclone config userinfo remote: [flags]

Options

@@ -2341,7 +2408,7 @@

See Also

rclone copyto

Copy files from source to dest, skipping identical files.

-

Synopsis

+

Synopsis

If source:path is a file or directory then it copies it to a file or directory named dest:path.

This can be used to upload single files to other than their current name. If the source is a directory then it acts exactly like the copy command.

So

@@ -2433,13 +2500,13 @@

See Also

rclone copyurl

Copy the contents of the URL supplied content to dest:path.

-

Synopsis

+

Synopsis

Download a URL's content and copy it to the destination without saving it in temporary storage.

Setting --auto-filename will attempt to automatically determine the filename from the URL (after any redirections) and used in the destination path.

With --auto-filename-header in addition, if a specific filename is set in HTTP headers, it will be used instead of the name from the URL. With --print-filename in addition, the resulting file name will be printed.

Setting --no-clobber will prevent overwriting file on the destination if there is one with the same name.

Setting --stdout or making the output file name - will cause the output to be written to standard output.

-

Troublshooting

+

Troubleshooting

If you can't get rclone copyurl to work then here are some things you can try:

  • --disable-http2 rclone will use HTTP2 if available - try disabling it
  • @@ -2468,7 +2535,7 @@

    See Also

rclone cryptcheck

Cryptcheck checks the integrity of an encrypted remote.

-

Synopsis

+

Synopsis

Checks a remote against a crypted remote. This is the equivalent of running rclone check, but able to check the checksums of the encrypted remote.

For it to work the underlying remote of the cryptedremote must support some kind of checksum.

It works by reading the nonce from each file on the cryptedremote: and using that to encrypt each file on the remote:. It then checks the checksum of the underlying file on the cryptedremote: against the checksum of the file it has just encrypted.

@@ -2536,7 +2603,7 @@

See Also

rclone cryptdecode

Cryptdecode returns unencrypted file names.

-

Synopsis

+

Synopsis

Returns unencrypted file names when provided with a list of encrypted file names. List limit is 10 items.

If you supply the --reverse flag, it will return encrypted file names.

use it like this

@@ -2555,7 +2622,7 @@

See Also

rclone deletefile

Remove a single file from remote.

-

Synopsis

+

Synopsis

Remove a single file from remote. Unlike delete it cannot be used to remove a directory and it doesn't obey include/exclude filters - if the specified file exists, it will always be removed.

rclone deletefile remote:path [flags]

Options

@@ -2572,7 +2639,7 @@

See Also

rclone gendocs

Output markdown docs for rclone to the directory supplied.

-

Synopsis

+

Synopsis

This produces markdown docs for the rclone commands to the directory supplied. These are in a format suitable for hugo to render into the rclone.org website.

rclone gendocs output_directory [flags]

Options

@@ -2584,36 +2651,36 @@

See Also

rclone gitannex

Speaks with git-annex over stdin/stdout.

-

Synopsis

+

Synopsis

Rclone's gitannex subcommand enables git-annex to store and retrieve content from an rclone remote. It is meant to be run by git-annex, not directly by users.

Installation on Linux

  1. Skip this step if your version of git-annex is 10.20240430 or newer. Otherwise, you must create a symlink somewhere on your PATH with a particular name. This symlink helps git-annex tell rclone it wants to run the "gitannex" subcommand.

    -
    # Create the helper symlink in "$HOME/bin".
    -ln -s "$(realpath rclone)" "$HOME/bin/git-annex-remote-rclone-builtin"
    -
    -# Verify the new symlink is on your PATH.
    -which git-annex-remote-rclone-builtin
  2. +
    # Create the helper symlink in "$HOME/bin".
    +ln -s "$(realpath rclone)" "$HOME/bin/git-annex-remote-rclone-builtin"
    +
    +# Verify the new symlink is on your PATH.
    +which git-annex-remote-rclone-builtin
  3. Add a new remote to your git-annex repo. This new remote will connect git-annex with the rclone gitannex subcommand.

    Start by asking git-annex to describe the remote's available configuration parameters.

    -
    # If you skipped step 1:
    -git annex initremote MyRemote type=rclone --whatelse
    -
    -# If you created a symlink in step 1:
    -git annex initremote MyRemote type=external externaltype=rclone-builtin --whatelse
    +
    # If you skipped step 1:
    +git annex initremote MyRemote type=rclone --whatelse
    +
    +# If you created a symlink in step 1:
    +git annex initremote MyRemote type=external externaltype=rclone-builtin --whatelse

    NOTE: If you're porting an existing git-annex-remote-rclone remote to use rclone gitannex, you can probably reuse the configuration parameters verbatim without renaming them. Check parameter synonyms with --whatelse as shown above.

    The following example creates a new git-annex remote named "MyRemote" that will use the rclone remote named "SomeRcloneRemote". That rclone remote must be one configured in your rclone.conf file, which can be located with rclone config file.

    -
    git annex initremote MyRemote         \
    -    type=external                     \
    -    externaltype=rclone-builtin       \
    -    encryption=none                   \
    -    rcloneremotename=SomeRcloneRemote \
    -    rcloneprefix=git-annex-content    \
    -    rclonelayout=nodir
  4. +
    git annex initremote MyRemote         \
    +    type=external                     \
    +    externaltype=rclone-builtin       \
    +    encryption=none                   \
    +    rcloneremotename=SomeRcloneRemote \
    +    rcloneprefix=git-annex-content    \
    +    rclonelayout=nodir
  5. Before you trust this command with your precious data, be sure to test the remote. This command is very new and has not been tested on many rclone backends. Caveat emptor!

    -
    git annex testremote MyRemote
  6. +
    git annex testremote MyRemote

Happy annexing!

rclone gitannex [flags]
@@ -2626,7 +2693,7 @@

See Also

rclone hashsum

Produces a hashsum file for all the objects in the path.

-

Synopsis

+

Synopsis

Produces a hash file for all the objects in the path using the hash named. The output is in the same format as the standard md5sum/sha1sum tool.

By default, the hash is requested from the remote. If the hash is not supported by the remote, no hash will be returned. With the download flag, the file will be downloaded from the remote and hashed locally enabling any hash for any remote.

For the MD5 and SHA1 algorithms there are also dedicated commands, md5sum and sha1sum.

@@ -2684,7 +2751,7 @@

See Also

rclone link

Generate public link to file/folder.

-

Synopsis

+

Synopsis

Create, retrieve or remove a public link to the given file or folder.

rclone link remote:path/to/file
 rclone link remote:path/to/folder/
@@ -2705,7 +2772,7 @@ 

See Also

rclone listremotes

List all the remotes in the config file and defined in environment variables.

-

Synopsis

+

Synopsis

Lists all the available remotes from the config file, or the remotes matching an optional filter.

Prints the result in human-readable format by default, and as a simple list of remote names, or if used with flag --long a tabular format including the remote names, types and descriptions. Using flag --json produces machine-readable output instead, which always includes all attributes - including the source (file or environment).

Result can be filtered by a filter argument which applies to all attributes, and/or filter flags specific for each attribute. The values must be specified according to regular rclone filtering pattern syntax.

@@ -2726,7 +2793,7 @@

See Also

rclone lsf

List directories and objects in remote:path formatted for parsing.

-

Synopsis

+

Synopsis

List the contents of the source path (directories and objects) to standard output in a form which is easy to parse by scripts. By default this will just be the names of the objects and directories, one per line. The directories will have a / suffix.

Eg

$ rclone lsf swift:bucket
@@ -2852,7 +2919,7 @@ 

See Also

rclone lsjson

List directories and objects in the path in JSON format.

-

Synopsis

+

Synopsis

List directories and objects in the path in JSON format.

The output is an array of Items, where each Item looks like this:

{
@@ -2958,7 +3025,7 @@ 

See Also

rclone mount

Mount the remote as file system on a mountpoint.

-

Synopsis

+

Synopsis

Rclone mount allows Linux, FreeBSD, macOS and Windows to mount any of Rclone's cloud storage systems as a file system with FUSE.

First set up your remote using rclone config. Check it works with rclone ls etc.

On Linux and macOS, you can run mount in either foreground or background (aka daemon) mode. Mount runs in foreground mode by default. Use the --daemon flag to force background mode. On Windows you can run mount in foreground only, the flag is ignored.

@@ -3328,7 +3395,7 @@

See Also

rclone moveto

Move file or directory from source to dest.

-

Synopsis

+

Synopsis

If source:path is a file or directory then it moves it to a file or directory named dest:path.

This can be used to rename files or upload single files to other than their existing name. If the source is a directory then it acts exactly like the move command.

So

@@ -3421,7 +3488,7 @@

See Also

rclone ncdu

Explore a remote with a text based user interface.

-

Synopsis

+

Synopsis

This displays a text based user interface allowing the navigation of a remote. It is most useful for answering the question - "What is using all my disk space?".

To make the user interface it first scans the entire remote given and builds an in memory representation. rclone ncdu can be used during this scanning phase and you will see it building up the directory structure as it goes along.

You can interact with the user interface using key presses, press '?' to toggle the help on and off. The supported keys are:

@@ -3497,7 +3564,7 @@

See Also

rclone nfsmount

Mount the remote as file system on a mountpoint.

-

Synopsis

+

Synopsis

Rclone nfsmount allows Linux, FreeBSD, macOS and Windows to mount any of Rclone's cloud storage systems as a file system with FUSE.

First set up your remote using rclone config. Check it works with rclone ls etc.

On Linux and macOS, you can run mount in either foreground or background (aka daemon) mode. Mount runs in foreground mode by default. Use the --daemon flag to force background mode. On Windows you can run mount in foreground only, the flag is ignored.

@@ -3872,7 +3939,7 @@

See Also

rclone obscure

Obscure password for use in the rclone config file.

-

Synopsis

+

Synopsis

In the rclone config file, human-readable passwords are obscured. Obscuring them is done by encrypting them and writing them out in base64. This is not a secure way of encrypting these passwords as rclone can decrypt them - it is to prevent "eyedropping" - namely someone seeing a password in the rclone config file by accident.

Many equally important things (like access tokens) are not obscured in the config file. However it is very hard to shoulder surf a 64 character hex token.

This command can also accept a password through STDIN instead of an argument by passing a hyphen as an argument. This will use the first line of STDIN as the password not including the trailing newline.

@@ -3889,7 +3956,7 @@

See Also

rclone rc

Run a command against a running rclone.

-

Synopsis

+

Synopsis

This runs a command against a running rclone. Use the --url flag to specify an non default URL to connect on. This can be either a ":port" which is taken to mean "http://localhost:port" or a "host:port" which is taken to mean "http://host:port"

A username and password can be passed in with --user and --pass.

Note that --rc-addr, --rc-user, --rc-pass will be read also for --url, --user, --pass.

@@ -3931,7 +3998,7 @@

See Also

rclone rcat

Copies standard input to file on remote.

-

Synopsis

+

Synopsis

Reads from standard input (stdin) and copies it to a single remote file.

echo "hello world" | rclone rcat remote:path/to/file
 ffmpeg - | rclone rcat remote:path/to/file
@@ -3956,7 +4023,7 @@

See Also

rclone rcd

Run rclone listening to remote control commands only.

-

Synopsis

+

Synopsis

This runs rclone so that it only listens to remote control commands.

This is useful if you are controlling rclone via the rc API.

If you pass in a path to a directory, rclone will serve that directory for GET requests on the URL passed in. It will also open the URL in the browser when rclone is run.

@@ -4142,7 +4209,7 @@

See Also

rclone rmdirs

Remove empty directories under the path.

-

Synopsis

+

Synopsis

This recursively removes any empty directories (including directories that only contain empty directories), that it finds under the path. The root path itself will also be removed if it is empty, unless you supply the --leave-root flag.

Use command rmdir to delete just the empty directory given by path, not recurse.

This is useful for tidying up remotes that rclone has left a lot of empty directories in. For example the delete command will delete files but leave the directory structure (unless used with option --rmdirs).

@@ -4164,7 +4231,7 @@

See Also

rclone selfupdate

Update the rclone binary.

-

Synopsis

+

Synopsis

This command downloads the latest release of rclone and replaces the currently running binary. The download is verified with a hashsum and cryptographically signed signature; see the release signing docs for details.

If used without flags (or with implied --stable flag), this command will install the latest stable release. However, some issues may be fixed (or features added) only in the latest beta release. In such cases you should run the command with the --beta flag, i.e. rclone selfupdate --beta. You can check in advance what version would be installed by adding the --check flag, then repeat the command without it when you are satisfied.

Sometimes the rclone team may recommend you a concrete beta or stable rclone release to troubleshoot your issue or add a bleeding edge feature. The --version VER flag, if given, will update to the concrete version instead of the latest one. If you omit micro version from VER (for example 1.53), the latest matching micro version will be used.

@@ -4189,7 +4256,7 @@

See Also

rclone serve

Serve a remote over a protocol.

-

Synopsis

+

Synopsis

Serve a remote over a given protocol. Requires the use of a subcommand to specify the protocol, e.g.

rclone serve http remote:

Each subcommand has its own options which you can see in their help.

@@ -4212,7 +4279,7 @@

See Also

rclone serve dlna

Serve remote:path over DLNA

-

Synopsis

+

Synopsis

Run a DLNA media server for media stored in an rclone remote. Many devices, such as the Xbox and PlayStation, can automatically discover this server in the LAN and play audio/video from it. VLC is also supported. Service discovery uses UDP multicast packets (SSDP) and will thus only work on LANs.

Rclone will list all files present in the remote, without filtering based on media formats or file extensions. Additionally, there is no media transcoding support. This means that some players might show files that they are not able to play back correctly.

Rclone will add external subtitle files (.srt) to videos if they have the same filename as the video file itself (except the extension), either in the same directory as the video, or in a "Subs" subdirectory.

@@ -4435,7 +4502,7 @@

See Also

rclone serve docker

Serve any remote on docker's volume plugin API.

-

Synopsis

+

Synopsis

This command implements the Docker volume plugin API allowing docker to use rclone as a data storage mechanism for various cloud providers. rclone provides docker volume plugin based on it.

To create a docker plugin, one must create a Unix or TCP socket that Docker will look for when you use the plugin and then it listens for commands from docker daemon and runs the corresponding code when necessary. Docker plugins can run as a managed plugin under control of the docker daemon or as an independent native service. For testing, you can just run it directly from the command line, for example:

sudo rclone serve docker --base-dir /tmp/rclone-volumes --socket-addr localhost:8787 -vv
@@ -4679,7 +4746,7 @@

See Also

rclone serve ftp

Serve remote:path over FTP.

-

Synopsis

+

Synopsis

Run a basic FTP server to serve a remote over FTP protocol. This can be viewed with a FTP client or you can make a remote of type FTP to read and write it.

Server options

Use --addr to specify which IP address and port the server should listen on, e.g. --addr 1.2.3.4:8000 or --addr :8080 to listen to all IPs. By default it only listens on localhost. You can use port :0 to let the OS choose an available port.

@@ -4935,7 +5002,7 @@

See Also

rclone serve http

Serve the remote over HTTP.

-

Synopsis

+

Synopsis

Run a basic web server to serve a remote over HTTP. This can be viewed in a web browser or you can make a remote of type http read from it.

You can use the filter flags (e.g. --include, --exclude) to control what is served.

The server will log errors. Use -v to see access logs.

@@ -5337,7 +5404,7 @@

See Also

rclone serve nfs

Serve the remote as an NFS mount

-

Synopsis

+

Synopsis

Create an NFS server that serves the given remote over the network.

This implements an NFSv3 server to serve any rclone remote via NFS.

The primary purpose for this command is to enable the mount command on recent macOS versions where installing FUSE is very cumbersome.

@@ -5346,7 +5413,7 @@

Synopsis

Modifying files through the NFS protocol requires VFS caching. Usually you will need to specify --vfs-cache-mode in order to be able to write to the mountpoint (full is recommended). If you don't specify VFS cache mode, the mount will be read-only.

--nfs-cache-type controls the type of the NFS handle cache. By default this is memory where new handles will be randomly allocated when needed. These are stored in memory. If the server is restarted the handle cache will be lost and connected NFS clients will get stale handle errors.

--nfs-cache-type disk uses an on disk NFS handle cache. Rclone hashes the path of the object and stores it in a file named after the hash. These hashes are stored on disk the directory controlled by --cache-dir or the exact directory may be specified with --nfs-cache-dir. Using this means that the NFS server can be restarted at will without affecting the connected clients.

-

--nfs-cache-type symlink is similar to --nfs-cache-type disk in that it uses an on disk cache, but the cache entries are held as symlinks. Rclone will use the handle of the underlying file as the NFS handle which improves performance. This sort of cache can't be backed up and restored as the underlying handles will change. This is Linux only. It requres running rclone as root or with CAP_DAC_READ_SEARCH. You can run rclone with this extra permission by doing this to the rclone binary sudo setcap cap_dac_read_search+ep /path/to/rclone.

+

--nfs-cache-type symlink is similar to --nfs-cache-type disk in that it uses an on disk cache, but the cache entries are held as symlinks. Rclone will use the handle of the underlying file as the NFS handle which improves performance. This sort of cache can't be backed up and restored as the underlying handles will change. This is Linux only. It requires running rclone as root or with CAP_DAC_READ_SEARCH. You can run rclone with this extra permission by doing this to the rclone binary sudo setcap cap_dac_read_search+ep /path/to/rclone.

--nfs-cache-handle-limit controls the maximum number of cached NFS handles stored by the caching handler. This should not be set too low or you may experience errors when trying to access files. The default is 1000000, but consider lowering this limit if the server's system resource usage causes problems. This is only used by the memory type cache.

To serve NFS over the network use following command:

rclone serve nfs remote: --addr 0.0.0.0:$PORT --vfs-cache-mode=full
@@ -5568,7 +5635,7 @@

See Also

rclone serve restic

Serve the remote for restic's REST API.

-

Synopsis

+

Synopsis

Run a basic web server to serve a remote over restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly.

Restic is a command-line program for doing backups.

The server will log errors. Use -v to see access logs.

@@ -5667,7 +5734,7 @@

See Also

rclone serve s3

Serve remote:path over s3.

-

Synopsis

+

Synopsis

serve s3 implements a basic s3 server that serves a remote via s3. This can be viewed with an s3 client, or you can make an s3 type remote to read and write to it with rclone.

serve s3 is considered Experimental so use with care.

S3 server supports Signature Version 4 authentication. Just use --auth-key accessKey,secretKey and set the Authorization header correctly in the request. (See the AWS docs).

@@ -5693,7 +5760,7 @@

Quickstart

access_key_id = ACCESS_KEY_ID secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false
-

Note that setting disable_multipart_uploads = true is to work around a bug which will be fixed in due course.

+

Note that setting use_multipart_uploads = false is to work around a bug which will be fixed in due course.

Bugs

When uploading multipart files serve s3 holds all the parts in memory (see #7453). This is a limitaton of the library rclone uses for serving S3 and will hopefully be fixed at some point.

Multipart server side copies do not work (see #7454). These take a very long time and eventually fail. The default threshold for multipart server side copies is 5G which is the maximum it can be, so files above this side will fail to be server side copied.

@@ -5988,7 +6055,7 @@

See Also

rclone serve sftp

Serve the remote over SFTP.

-

Synopsis

+

Synopsis

Run an SFTP server to serve a remote over SFTP. This can be used with an SFTP client or you can make a remote of type sftp to use with it.

You can use the filter flags (e.g. --include, --exclude) to control what is served.

The server will respond to a small number of shell commands, mainly md5sum, sha1sum and df, which enable it to provide support for checksums and the about feature when accessed from an sftp remote.

@@ -6255,7 +6322,7 @@

See Also

rclone serve webdav

Serve remote:path over WebDAV.

-

Synopsis

+

Synopsis

Run a basic WebDAV server to serve a remote over HTTP via the WebDAV protocol. This can be viewed with a WebDAV client, through a web browser, or you can make a remote of type WebDAV to read and write it.

WebDAV options

--etag-hash

@@ -6671,7 +6738,7 @@

See Also

rclone settier

Changes storage class/tier of objects in remote.

-

Synopsis

+

Synopsis

Changes storage tier or class at remote if supported. Few cloud storage services provides different storage classes on objects, for example AWS S3 and Glacier, Azure Blob storage - Hot, Cool and Archive, Google Cloud Storage, Regional Storage, Nearline, Coldline etc.

Note that, certain tier changes make objects not available to access immediately. For example tiering to archive in azure blob storage makes objects in frozen state, user can restore by setting tier to Hot/Cool, similarly S3 to Glacier makes object inaccessible.true

You can use it to tier single object

@@ -6690,7 +6757,7 @@

See Also

rclone test

Run a test command

-

Synopsis

+

Synopsis

Rclone test is used to run test commands.

Select which test command you want with the subcommand, eg

rclone test memory remote:
@@ -6722,7 +6789,7 @@

See Also

rclone test histogram

Makes a histogram of file name characters.

-

Synopsis

+

Synopsis

This command outputs JSON which shows the histogram of characters used in filenames in the remote:path specified.

The data doesn't contain any identifying information but is useful for the rclone developers when developing filename compression.

rclone test histogram [remote:path] [flags]
@@ -6735,7 +6802,7 @@

See Also

rclone test info

Discovers file name or other limitations for paths.

-

Synopsis

+

Synopsis

Discovers what filenames and upload methods are possible to write to the paths passed in and how long they can be. It can take some time. It will write test files into the remote:path passed in. It outputs a bit of go code for each one.

NB this can create undeletable files and other hazards - use with care

rclone test info [remote:path]+ [flags]
@@ -6807,7 +6874,7 @@

See Also

rclone touch

Create new file or change file modification time.

-

Synopsis

+

Synopsis

Set the modification time on file(s) as specified by remote:path to have the current time.

If remote:path does not exist then a zero sized file will be created, unless --no-create or --recursive is provided.

If --recursive is used then recursively sets the modification time on all existing files that is found under the path. Filters are supported, and you can test with the --dry-run or the --interactive/-i flag.

@@ -6865,7 +6932,7 @@

See Also

rclone tree

List the contents of the remote in a tree like fashion.

-

Synopsis

+

Synopsis

Lists the contents of a remote in a similar way to the unix tree command.

For example

$ rclone tree remote:path
@@ -7168,6 +7235,7 @@ 

Standard system metadata

Options

Rclone has a number of options to control its behaviour.

Options that take parameters can have the values passed in two ways, --option=value or --option value. However boolean (true/false) options behave slightly differently to the other options in that --boolean sets the option to true and the absence of the flag sets it to false. It is also possible to specify --boolean=false or --boolean=true. Note that --boolean false is not valid - this is parsed as --boolean and the false is parsed as an extra command line argument for rclone.

+

Options documented to take a stringArray parameter accept multiple values. To pass more than one value, repeat the option; for example: --include value1 --include value2.

Time or duration options

TIME or DURATION options can be specified as a duration string or a time string.

A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". Default units are seconds or the following abbreviations are valid:

@@ -7530,55 +7598,55 @@

--metadata-mapper SpaceSepList

  • ID is the source ID of the object if known.
  • Metadata is the backend specific metadata as described in the backend docs.
  • -
    {
    -    "SrcFs": "gdrive:",
    -    "SrcFsType": "drive",
    -    "DstFs": "newdrive:user",
    -    "DstFsType": "onedrive",
    -    "Remote": "test.txt",
    -    "Size": 6,
    -    "MimeType": "text/plain; charset=utf-8",
    -    "ModTime": "2022-10-11T17:53:10.286745272+01:00",
    -    "IsDir": false,
    -    "ID": "xyz",
    -    "Metadata": {
    -        "btime": "2022-10-11T16:53:11Z",
    -        "content-type": "text/plain; charset=utf-8",
    -        "mtime": "2022-10-11T17:53:10.286745272+01:00",
    -        "owner": "user1@domain1.com",
    -        "permissions": "...",
    -        "description": "my nice file",
    -        "starred": "false"
    -    }
    -}
    -

    The program should then modify the input as desired and send it to STDOUT. The returned Metadata field will be used in its entirety for the destination object. Any other fields will be ignored. Note in this example we translate user names and permissions and add something to the description:

    {
    -    "Metadata": {
    -        "btime": "2022-10-11T16:53:11Z",
    -        "content-type": "text/plain; charset=utf-8",
    -        "mtime": "2022-10-11T17:53:10.286745272+01:00",
    -        "owner": "user1@domain2.com",
    -        "permissions": "...",
    -        "description": "my nice file [migrated from domain1]",
    -        "starred": "false"
    -    }
    -}
    + "SrcFs": "gdrive:", + "SrcFsType": "drive", + "DstFs": "newdrive:user", + "DstFsType": "onedrive", + "Remote": "test.txt", + "Size": 6, + "MimeType": "text/plain; charset=utf-8", + "ModTime": "2022-10-11T17:53:10.286745272+01:00", + "IsDir": false, + "ID": "xyz", + "Metadata": { + "btime": "2022-10-11T16:53:11Z", + "content-type": "text/plain; charset=utf-8", + "mtime": "2022-10-11T17:53:10.286745272+01:00", + "owner": "user1@domain1.com", + "permissions": "...", + "description": "my nice file", + "starred": "false" + } +}
    +

    The program should then modify the input as desired and send it to STDOUT. The returned Metadata field will be used in its entirety for the destination object. Any other fields will be ignored. Note in this example we translate user names and permissions and add something to the description:

    +
    {
    +    "Metadata": {
    +        "btime": "2022-10-11T16:53:11Z",
    +        "content-type": "text/plain; charset=utf-8",
    +        "mtime": "2022-10-11T17:53:10.286745272+01:00",
    +        "owner": "user1@domain2.com",
    +        "permissions": "...",
    +        "description": "my nice file [migrated from domain1]",
    +        "starred": "false"
    +    }
    +}

    Metadata can be removed here too.

    An example python program might look something like this to implement the above transformations.

    -
    import sys, json
    -
    -i = json.load(sys.stdin)
    -metadata = i["Metadata"]
    -# Add tag to description
    -if "description" in metadata:
    -    metadata["description"] += " [migrated from domain1]"
    -else:
    -    metadata["description"] = "[migrated from domain1]"
    -# Modify owner
    -if "owner" in metadata:
    -    metadata["owner"] = metadata["owner"].replace("domain1.com", "domain2.com")
    -o = { "Metadata": metadata }
    -json.dump(o, sys.stdout, indent="\t")
    +
    import sys, json
    +
    +i = json.load(sys.stdin)
    +metadata = i["Metadata"]
    +# Add tag to description
    +if "description" in metadata:
    +    metadata["description"] += " [migrated from domain1]"
    +else:
    +    metadata["description"] = "[migrated from domain1]"
    +# Modify owner
    +if "owner" in metadata:
    +    metadata["owner"] = metadata["owner"].replace("domain1.com", "domain2.com")
    +o = { "Metadata": metadata }
    +json.dump(o, sys.stdout, indent="\t")

    You can find this example (slightly expanded) in the rclone source code at bin/test_metadata_mapper.py.

    If you want to see the input to the metadata mapper and the output returned from it in the log you can use -vv --dump mapper.

    See the metadata section for more info.

    @@ -8051,7 +8119,7 @@

    Options

    Verbosity is slightly different, the environment variable equivalent of --verbose or -v is RCLONE_VERBOSE=1, or for -vv, RCLONE_VERBOSE=2.

    The same parser is used for the options and the environment variables so they take exactly the same form.

    The options set by environment variables can be seen with the -vv flag, e.g. rclone version -vv.

    -

    Options that can appear multiple times (type stringArray) are treated slighly differently as environment variables can only be defined once. In order to allow a simple mechanism for adding one or many items, the input is treated as a CSV encoded string. For example

    +

    Options that can appear multiple times (type stringArray) are treated slightly differently as environment variables can only be defined once. In order to allow a simple mechanism for adding one or many items, the input is treated as a CSV encoded string. For example

    @@ -10160,7 +10228,7 @@

    vfs/queue: Queue info for a VFS.

    }, ], } -

    The expiry time is the time until the file is elegible for being uploaded in floating point seconds. This may go negative. As rclone only transfers --transfers files at once, only the lowest --transfers expiry times will have uploading as true. So there may be files with negative expiry times for which uploading is false.

    +

    The expiry time is the time until the file is eligible for being uploaded in floating point seconds. This may go negative. As rclone only transfers --transfers files at once, only the lowest --transfers expiry times will have uploading as true. So there may be files with negative expiry times for which uploading is false.

    This command takes an "fs" parameter. If this parameter is not supplied and if there is only one VFS in use then that VFS will be used. If there is more than one VFS in use then the "fs" parameter must be supplied.

    vfs/queue-set-expiry: Set the expiry time for an item queued for upload.

    Use this to adjust the expiry time for an item in the upload queue. You will need to read the id of the item using vfs/queue before using this call.

    @@ -12221,7 +12289,7 @@

    Networking

    --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.1")

    Performance

    Flags helpful for increasing performance.

          --buffer-size SizeSuffix   In memory buffer size when reading files for each --transfer (default 16Mi)
    @@ -13319,7 +13387,7 @@ 

    Running Plugin under Systemd

    systemctl restart docker

    Or run the service directly: - run systemctl daemon-reload to let systemd pick up new config - run systemctl enable docker-volume-rclone.service to make the new service start automatically when you power on your machine. - run systemctl start docker-volume-rclone.service to start the service now. - run systemctl restart docker to restart docker daemon and let it detect the new plugin socket. Note that this step is not needed in managed mode where docker knows about plugin state changes.

    The two methods are equivalent from the user perspective, but I personally prefer socket activation.

    -

    Troubleshooting

    +

    Troubleshooting

    You can see managed plugin settings with

    docker plugin list
     docker plugin inspect rclone
    @@ -13516,7 +13584,7 @@

    --max-delete

    As a safety check, if greater than the --max-delete percent of files were deleted on either the Path1 or Path2 filesystem, then bisync will abort with a warning message, without making any changes. The default --max-delete is 50%. One way to trigger this limit is to rename a directory that contains more than half of your files. This will appear to bisync as a bunch of deleted files and a bunch of new files. This safety check is intended to block bisync from deleting all of the files on both filesystems due to a temporary network access issue, or if the user had inadvertently deleted the files on one side or the other. To force the sync, either set a different delete percentage limit, e.g. --max-delete 75 (allows up to 75% deletion), or use --force to bypass the check.

    Also see the all files changed check.

    --filters-file

    -

    By using rclone filter features you can exclude file types or directory sub-trees from the sync. See the bisync filters section and generic --filter-from documentation. An example filters file contains filters for non-allowed files for synching with Dropbox.

    +

    By using rclone filter features you can exclude file types or directory sub-trees from the sync. See the bisync filters section and generic --filter-from documentation. An example filters file contains filters for non-allowed files for syncing with Dropbox.

    If you make changes to your filters file then bisync requires a run with --resync. This is a safety feature, which prevents existing files on the Path1 and/or Path2 side from seeming to disappear from view (since they are excluded in the new listings), which would fool bisync into seeing them as deleted (as compared to the prior run listings), and then bisync would proceed to delete them for real.

    To block this from happening, bisync calculates an MD5 hash of the filters file and stores the hash in a .md5 file in the same place as your filters file. On the next run with --filters-file set, bisync re-calculates the MD5 hash of the current filters file and compares it to the hash stored in the .md5 file. If they don't match, the run aborts with a critical error and thus forces you to do a --resync, likely avoiding a disaster.

    --conflict-resolve CHOICE

    @@ -13556,7 +13624,7 @@

    --conflict-suffix STRING[,STRING]

    --check-sync

    Enabled by default, the check-sync function checks that all of the same files exist in both the Path1 and Path2 history listings. This check-sync integrity check is performed at the end of the sync run by default. Any untrapped failing copy/deletes between the two paths might result in differences between the two listings and in the untracked file content differences between the two paths. A resync run would correct the error.

    Note that the default-enabled integrity check locally executes a load of both the final Path1 and Path2 listings, and thus adds to the run time of a sync. Using --check-sync=false will disable it and may significantly reduce the sync run times for very large numbers of files.

    -

    The check may be run manually with --check-sync=only. It runs only the integrity check and terminates without actually synching.

    +

    The check may be run manually with --check-sync=only. It runs only the integrity check and terminates without actually syncing.

    Note that currently, --check-sync only checks listing snapshots and NOT the actual files on the remotes. Note also that the listing snapshots will not know about any changes that happened during or after the latest bisync run, as those will be discovered on the next run. Therefore, while listings should always match each other at the end of a bisync run, it is expected that they will not match the underlying remotes, nor will the remotes match each other, if there were changes during or after the run. This is normal, and any differences will be detected and synced on the next run.

    For a robust integrity check of the current state of the remotes (as opposed to just their listing snapshots), consider using check (or cryptcheck, if at least one path is a crypt remote) instead of --check-sync, keeping in mind that differences are expected if files changed during or after your last bisync run.

    For example, a possible sequence could look like this:

    @@ -13786,7 +13854,7 @@

    Filtering

    See filtering documentation for how filter rules are written and interpreted.

    Bisync's --filters-file flag slightly extends the rclone's --filter-from filtering mechanism. For a given bisync run you may provide only one --filters-file. The --include*, --exclude*, and --filter flags are also supported.

    How to filter directories

    -

    Filtering portions of the directory tree is a critical feature for synching.

    +

    Filtering portions of the directory tree is a critical feature for syncing.

    Examples of directory trees (always beneath the Path1/Path2 root level) you may want to exclude from your sync: - Directory trees containing only software build intermediate files. - Directory trees containing application temporary files and data such as the Windows C:\Users\MyLogin\AppData\ tree. - Directory trees containing files that are large, less important, or are getting thrashed continuously by ongoing processes.

    On the other hand, there may be only select directories that you actually want to sync, and exclude all others. See the Example include-style filters for Windows user directories below.

    Filters file writing guidelines

    @@ -13853,7 +13921,7 @@

    Example include-style filters for Windows user director

    Note also that Windows implements several "library" links such as C:\Users\MyLogin\My Documents\My Music pointing to C:\Users\MyLogin\Music. rclone sees these as links, so you must add --links to the bisync command line if you which to follow these links. I find that I get permission errors in trying to follow the links, so I don't include the rclone --links flag, but then you get lots of Can't follow symlink… noise from rclone about not following the links. This noise can be quashed by adding --quiet to the bisync command line.

    Example exclude-style filters files for use with Dropbox

      -
    • Dropbox disallows synching the listed temporary and configuration/data files. The `- ` filters exclude these files where ever they may occur in the sync tree. Consider adding similar exclusions for file types you don't need to sync, such as core dump and software build files.
    • +
    • Dropbox disallows syncing the listed temporary and configuration/data files. The `- ` filters exclude these files where ever they may occur in the sync tree. Consider adding similar exclusions for file types you don't need to sync, such as core dump and software build files.
    • bisync testing creates /testdir/ at the top level of the sync tree, and usually deletes the tree after the test. If a normal sync should run while the /testdir/ tree exists the --check-access phase may fail due to unbalanced RCLONE_TEST files. The `- /testdir/` filter blocks this tree from being synched. You don't need this exclusion if you are not doing bisync development testing.
    • Everything else beneath the Path1/Path2 root will be synched.
    • RCLONE_TEST files may be placed anywhere within the tree, including the root.
    • @@ -14013,7 +14081,7 @@

      Test command syntax

      Note: unlike rclone flags which must be prefixed by double dash (--), the test command flags can be equally prefixed by a single - or double dash.

      Running tests

        -
      • go test . -case basic -remote local -remote2 local runs the test_basic test case using only the local filesystem, synching one local directory with another local directory. Test script output is to the console, while commands within scenario.txt have their output sent to the .../workdir/test.log file, which is finally compared to the golden copy.
      • +
      • go test . -case basic -remote local -remote2 local runs the test_basic test case using only the local filesystem, syncing one local directory with another local directory. Test script output is to the console, while commands within scenario.txt have their output sent to the .../workdir/test.log file, which is finally compared to the golden copy.
      • The first argument after go test should be a relative name of the directory containing bisync source code. If you run tests right from there, the argument will be . (current directory) as in most examples below. If you run bisync tests from the rclone source directory, the command should be go test ./cmd/bisync ....
      • The test engine will mangle rclone output to ensure comparability with golden listings and logs.
      • Test scenarios are located in ./cmd/bisync/testdata. The test -case argument should match the full name of a subdirectory under that directory. Every test subdirectory name on disk must start with test_, this prefix can be omitted on command line for brevity. Also, underscores in the name can be replaced by dashes for convenience.
      • @@ -14150,6 +14218,10 @@

        References

        Bisync adopts the differential synchronization technique, which is based on keeping history of changes performed by both synchronizing sides. See the Dual Shadow Method section in Neil Fraser's article.

        Also note a number of academic publications by Benjamin Pierce about Unison and synchronization in general.

        Changelog

        +

        v1.69.1

        +
          +
        • Fixed an issue causing listings to not capture concurrent modifications under certain conditions
        • +

        v1.68

        • Fixed an issue affecting backends that round modtimes to a lower precision.
        • @@ -15072,7 +15144,7 @@

          S3 Permissions

          1. This is a policy that can be used when creating bucket. It assumes that USER_NAME has been created.
          2. The Resource entry must include both resource ARNs, as one implies the bucket and the other implies the bucket's objects.
          3. -
          4. When using s3-no-check-bucket and the bucket already exsits, the "arn:aws:s3:::BUCKET_NAME" doesn't have to be included.
          5. +
          6. When using s3-no-check-bucket and the bucket already exists, the "arn:aws:s3:::BUCKET_NAME" doesn't have to be included.

          For reference, here's an Ansible script that will generate one or more buckets that will work with rclone sync.

          Key Management System (KMS)

          @@ -17632,7 +17704,7 @@

          RackCorp

          endpoint = s3.rackcorp.com location_constraint = au-nsw

          Rclone Serve S3

          -

          Rclone can serve any remote over the S3 protocol. For details see the rclone serve s3 documentation.

          +

          Rclone can serve any remote over the S3 protocol. For details see the rclone serve s3 documentation.

          For example, to serve remote:path over s3, run the server like this:

          rclone serve s3 --auth-key ACCESS_KEY_ID,SECRET_ACCESS_KEY remote:path

          This will be compatible with an rclone remote which is defined like this:

          @@ -17643,7 +17715,7 @@

          Rclone Serve S3

          access_key_id = ACCESS_KEY_ID secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false -

          Note that setting disable_multipart_uploads = true is to work around a bug which will be fixed in due course.

          +

          Note that setting use_multipart_uploads = false is to work around a bug which will be fixed in due course.

          Scaleway

          Scaleway The Object Storage platform allows you to store anything from backups, logs and web assets to documents and photos. Files can be dropped from the Scaleway console or transferred through our API and CLI or using any S3-compatible tool.

          Scaleway provides an S3 interface which can be configured for use with rclone like this:

          @@ -18581,27 +18653,49 @@

          Linode

          Endpoint for Linode Object Storage API. Choose a number from below, or type in your own value. Press Enter to leave empty. - 1 / Atlanta, GA (USA), us-southeast-1 + 1 / Amsterdam (Netherlands), nl-ams-1 + \ (nl-ams-1.linodeobjects.com) + 2 / Atlanta, GA (USA), us-southeast-1 \ (us-southeast-1.linodeobjects.com) - 2 / Chicago, IL (USA), us-ord-1 + 3 / Chennai (India), in-maa-1 + \ (in-maa-1.linodeobjects.com) + 4 / Chicago, IL (USA), us-ord-1 \ (us-ord-1.linodeobjects.com) - 3 / Frankfurt (Germany), eu-central-1 + 5 / Frankfurt (Germany), eu-central-1 \ (eu-central-1.linodeobjects.com) - 4 / Milan (Italy), it-mil-1 + 6 / Jakarta (Indonesia), id-cgk-1 + \ (id-cgk-1.linodeobjects.com) + 7 / London 2 (Great Britain), gb-lon-1 + \ (gb-lon-1.linodeobjects.com) + 8 / Los Angeles, CA (USA), us-lax-1 + \ (us-lax-1.linodeobjects.com) + 9 / Madrid (Spain), es-mad-1 + \ (es-mad-1.linodeobjects.com) +10 / Melbourne (Australia), au-mel-1 + \ (au-mel-1.linodeobjects.com) +11 / Miami, FL (USA), us-mia-1 + \ (us-mia-1.linodeobjects.com) +12 / Milan (Italy), it-mil-1 \ (it-mil-1.linodeobjects.com) - 5 / Newark, NJ (USA), us-east-1 +13 / Newark, NJ (USA), us-east-1 \ (us-east-1.linodeobjects.com) - 6 / Paris (France), fr-par-1 +14 / Osaka (Japan), jp-osa-1 + \ (jp-osa-1.linodeobjects.com) +15 / Paris (France), fr-par-1 \ (fr-par-1.linodeobjects.com) - 7 / Seattle, WA (USA), us-sea-1 +16 / São Paulo (Brazil), br-gru-1 + \ (br-gru-1.linodeobjects.com) +17 / Seattle, WA (USA), us-sea-1 \ (us-sea-1.linodeobjects.com) - 8 / Singapore ap-south-1 +18 / Singapore, ap-south-1 \ (ap-south-1.linodeobjects.com) - 9 / Stockholm (Sweden), se-sto-1 +19 / Singapore 2, sg-sin-1 + \ (sg-sin-1.linodeobjects.com) +20 / Stockholm (Sweden), se-sto-1 \ (se-sto-1.linodeobjects.com) -10 / Washington, DC, (USA), us-iad-1 +21 / Washington, DC, (USA), us-iad-1 \ (us-iad-1.linodeobjects.com) -endpoint> 3 +endpoint> 5 Option acl. Canned ACL used when creating buckets and storing or copying objects. @@ -18958,7 +19052,7 @@

          Netease NOS

          For Netease NOS configure as per the configurator rclone config setting the provider Netease. This will automatically set force_path_style = false which is necessary for it to run properly.

          Petabox

          Here is an example of making a Petabox configuration. First run:

          -
          rclone config
          +
          rclone config

          This will guide you through an interactive setup process.

          No remotes found, make a new one?
           n) New remote
          @@ -21755,7 +21849,7 @@ 

          Header

        • 8 bytes magic string RCLONE\x00\x00
        • 24 bytes Nonce (IV)
        -

        The initial nonce is generated from the operating systems crypto strong random number generator. The nonce is incremented for each chunk read making sure each nonce is unique for each block written. The chance of a nonce being reused is minuscule. If you wrote an exabyte of data (10¹⁸ bytes) you would have a probability of approximately 2×10⁻³² of re-using a nonce.

        +

        The initial nonce is generated from the operating systems crypto strong random number generator. The nonce is incremented for each chunk read making sure each nonce is unique for each block written. The chance of a nonce being reused is minuscule. If you wrote an exabyte of data (10¹⁸ bytes) you would have a probability of approximately 2×10⁻³² of reusing a nonce.

        Chunk

        Each chunk will contain 64 KiB of data, except for the last one which may have less data. The data chunk is in standard NaCl SecretBox format. SecretBox uses XSalsa20 and Poly1305 to encrypt and authenticate messages.

        Each chunk contains:

        @@ -27047,7 +27141,7 @@

        Configuration

        config_2fa> 2FACODE Remote config -------------------- -[koofr] +[iclouddrive] - type: iclouddrive - apple_id: APPLEID - password: *** ENCRYPTED *** @@ -27060,6 +27154,13 @@

        Configuration

        y/e/d> y

        Advanced Data Protection

        ADP is currently unsupported and need to be disabled

        +

        On iPhone, Settings > Apple Account > iCloud > 'Access iCloud Data on the Web' must be ON, and 'Advanced Data Protection' OFF.

        +

        Troubleshooting

        +

        Missing PCS cookies from the request

        +

        This means you have Advanced Data Protection (ADP) turned on. This is not supported at the moment. If you want to use rclone you will have to turn it off. See above for how to turn it off.

        +

        You will need to clear the cookies and the trust_token fields in the config. Or you can delete the remote config and start again.

        +

        You should then run rclone reconnect remote:.

        +

        Note that changing the ADP setting may not take effect immediately - you may need to wait a few hours or a day before you can get rclone to work - keep clearing the config entry and running rclone reconnect remote: until rclone functions properly.

        Standard options

        Here are the Standard options specific to iclouddrive (iCloud Drive).

        --iclouddrive-apple-id

        @@ -27801,7 +27902,7 @@

        Limitations

        Note that Jottacloud is case insensitive so you can't have a file called "Hello.doc" and one called "hello.doc".

        There are quite a few characters that can't be in Jottacloud file names. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a ? in it will be mapped to ? instead.

        Jottacloud only supports filenames up to 255 characters in length.

        -

        Troubleshooting

        +

        Troubleshooting

        Jottacloud exhibits some inconsistent behaviours regarding deleted files and folders which may cause Copy, Move and DirMove operations to previously deleted paths to fail. Emptying the trash should help in such cases.

        Koofr

        Paths are specified as remote:path

        @@ -30461,7 +30562,7 @@

        --onedrive-region

    • "de"
        -
      • Microsoft Cloud Germany
      • +
      • Microsoft Cloud Germany (deprecated - try global region first).
    • "cn"
        @@ -30834,75 +30935,75 @@

        Metadata

        Permissions are also supported, if --onedrive-metadata-permissions is set. The accepted values for --onedrive-metadata-permissions are "read", "write", "read,write", and "off" (the default). "write" supports adding new permissions, updating the "role" of existing permissions, and removing permissions. Updating and removing require the Permission ID to be known, so it is recommended to use "read,write" instead of "write" if you wish to update/remove permissions.

        Permissions are read/written in JSON format using the same schema as the OneDrive API, which differs slightly between OneDrive Personal and Business.

        Example for OneDrive Personal:

        -
        [
        -    {
        -        "id": "1234567890ABC!123",
        -        "grantedTo": {
        -            "user": {
        -                "id": "ryan@contoso.com"
        -            },
        -            "application": {},
        -            "device": {}
        -        },
        -        "invitation": {
        -            "email": "ryan@contoso.com"
        -        },
        -        "link": {
        -            "webUrl": "https://1drv.ms/t/s!1234567890ABC"
        -        },
        -        "roles": [
        -            "read"
        -        ],
        -        "shareId": "s!1234567890ABC"
        -    }
        -]
        -

        Example for OneDrive Business:

        [
             {
        -        "id": "48d31887-5fad-4d73-a9f5-3c356e68a038",
        -        "grantedToIdentities": [
        -            {
        -                "user": {
        -                    "displayName": "ryan@contoso.com"
        -                },
        -                "application": {},
        -                "device": {}
        -            }
        -        ],
        -        "link": {
        -            "type": "view",
        -            "scope": "users",
        -            "webUrl": "https://contoso.sharepoint.com/:w:/t/design/a577ghg9hgh737613bmbjf839026561fmzhsr85ng9f3hjck2t5s"
        -        },
        -        "roles": [
        -            "read"
        -        ],
        -        "shareId": "u!LKj1lkdlals90j1nlkascl"
        -    },
        -    {
        -        "id": "5D33DD65C6932946",
        -        "grantedTo": {
        -            "user": {
        -                "displayName": "John Doe",
        -                "id": "efee1b77-fb3b-4f65-99d6-274c11914d12"
        -            },
        -            "application": {},
        -            "device": {}
        -        },
        -        "roles": [
        -            "owner"
        -        ],
        -        "shareId": "FWxc1lasfdbEAGM5fI7B67aB5ZMPDMmQ11U"
        -    }
        -]
        + "id": "1234567890ABC!123", + "grantedTo": { + "user": { + "id": "ryan@contoso.com" + }, + "application": {}, + "device": {} + }, + "invitation": { + "email": "ryan@contoso.com" + }, + "link": { + "webUrl": "https://1drv.ms/t/s!1234567890ABC" + }, + "roles": [ + "read" + ], + "shareId": "s!1234567890ABC" + } +] +

        Example for OneDrive Business:

        +
        [
        +    {
        +        "id": "48d31887-5fad-4d73-a9f5-3c356e68a038",
        +        "grantedToIdentities": [
        +            {
        +                "user": {
        +                    "displayName": "ryan@contoso.com"
        +                },
        +                "application": {},
        +                "device": {}
        +            }
        +        ],
        +        "link": {
        +            "type": "view",
        +            "scope": "users",
        +            "webUrl": "https://contoso.sharepoint.com/:w:/t/design/a577ghg9hgh737613bmbjf839026561fmzhsr85ng9f3hjck2t5s"
        +        },
        +        "roles": [
        +            "read"
        +        ],
        +        "shareId": "u!LKj1lkdlals90j1nlkascl"
        +    },
        +    {
        +        "id": "5D33DD65C6932946",
        +        "grantedTo": {
        +            "user": {
        +                "displayName": "John Doe",
        +                "id": "efee1b77-fb3b-4f65-99d6-274c11914d12"
        +            },
        +            "application": {},
        +            "device": {}
        +        },
        +        "roles": [
        +            "owner"
        +        ],
        +        "shareId": "FWxc1lasfdbEAGM5fI7B67aB5ZMPDMmQ11U"
        +    }
        +]

        To write permissions, pass in a "permissions" metadata key using this same format. The --metadata-mapper tool can be very helpful for this.

        When adding permissions, an email address can be provided in the User.ID or DisplayName properties of grantedTo or grantedToIdentities. Alternatively, an ObjectID can be provided in User.ID. At least one valid recipient must be provided in order to add a permission for a user. Creating a Public Link is also supported, if Link.Scope is set to "anonymous".

        Example request to add a "read" permission with --metadata-mapper:

        -
        {
        -    "Metadata": {
        -        "permissions": "[{\"grantedToIdentities\":[{\"user\":{\"id\":\"ryan@contoso.com\"}}],\"roles\":[\"read\"]}]"
        -    }
        -}
        +
        {
        +    "Metadata": {
        +        "permissions": "[{\"grantedToIdentities\":[{\"user\":{\"id\":\"ryan@contoso.com\"}}],\"roles\":[\"read\"]}]"
        +    }
        +}

        Note that adding a permission can fail if a conflicting permission already exists for the file/folder.

        To update an existing permission, include both the Permission ID and the new roles to be assigned. roles is the only property that can be changed.

        To remove permissions, pass in a blob containing only the permissions you wish to keep (which can be empty, to remove all.) Note that the owner role will be ignored, as it cannot be removed.

        @@ -31052,6 +31153,26 @@

        Metadata

    See the metadata docs for more info.

    +

    Impersonate other users as Admin

    +

    Unlike Google Drive and impersonating any domain user via service accounts, OneDrive requires you to authenticate as an admin account, and manually setup a remote per user you wish to impersonate.

    +
      +
    1. In Microsoft 365 Admin Center, open each user you need to "impersonate" and go to the OneDrive section. There is a heading called "Get access to files", you need to click to create the link, this creates the link of the format https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/ but also changes the permissions so you your admin user has access.
    2. +
    3. Then in powershell run the following commands:
    4. +
    +
    Install-Module Microsoft.Graph -Scope CurrentUser -Repository PSGallery -Force
    +Import-Module Microsoft.Graph.Files
    +Connect-MgGraph -Scopes "Files.ReadWrite.All"
    +# Follow the steps to allow access to your admin user
    +# Then run this for each user you want to impersonate to get the Drive ID
    +Get-MgUserDefaultDrive -UserId '{emailaddress}'
    +# This will give you output of the format:
    +# Name     Id                                                                 DriveType CreatedDateTime
    +# ----     --                                                                 --------- ---------------
    +# OneDrive b!XYZ123                                                           business  14/10/2023 1:00:58 pm
    +
    +
      +
    1. Then in rclone add a onedrive remote type, and use the Type in driveID with the DriveID you got in the previous step. One remote per user. It will then confirm the drive ID, and hopefully give you a message of Found drive "root" of type "business" and then include the URL of the format https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/Documents
    2. +

    Limitations

    If you don't use rclone for 90 days the refresh token will expire. This will result in authorization problems. This is easy to fix by running the rclone config reconnect remote: command to get a new token and refresh token.

    Naming

    @@ -31097,7 +31218,7 @@

    Cleanup

    rclone cleanup --interactive remote:path/subdir # interactively remove all old version for path/subdir
     rclone cleanup remote:path/subdir               # unconditionally remove all old version for path/subdir

    NB Onedrive personal can't currently delete versions

    -

    Troubleshooting

    +

    Troubleshooting

    Excessive throttling or blocked on SharePoint

    If you experience excessive throttling or is being blocked on SharePoint then it may help to set the user agent explicitly with a flag like this: --user-agent "ISV|rclone.org|rclone/v1.55.1"

    The specific details can be found in the Microsoft document: Avoid getting throttled or blocked in SharePoint Online

    @@ -33067,7 +33188,7 @@

    --swift-description

    Limitations

    The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these.

    -

    Troubleshooting

    +

    Troubleshooting

    Rclone gives Failed to create file system for "remote:": Bad Request

    Due to an oddity of the underlying swift library, it gives a "Bad Request" error rather than a more sensible error when the authentication fails for Swift.

    So this most likely means your username / password is wrong. You can investigate further with the --dump-bodies flag.

    @@ -38190,6 +38311,43 @@

    noop

  • "error": return an error based on option value
  • Changelog

    +

    v1.69.1 - 2025-02-14

    +

    See commits

    +
      +
    • Bug Fixes +
        +
      • lib/oauthutil: Fix redirect URL mismatch errors (Nick Craig-Wood)
      • +
      • bisync: Fix listings missing concurrent modifications (nielash)
      • +
      • serve s3: Fix list objects encoding-type (Nick Craig-Wood)
      • +
      • fs: Fix confusing "didn't find section in config file" error (Nick Craig-Wood)
      • +
      • doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt Ickstadt, Nick Craig-Wood, Tim White, Zachary Vorhies)
      • +
      • build: Added parallel docker builds and caching for go build in the container (Anagh Kumar Baranwal)
      • +
    • +
    • VFS +
        +
      • Fix the cache failing to upload symlinks when --links was specified (Nick Craig-Wood)
      • +
      • Fix race detected by race detector (Nick Craig-Wood)
      • +
      • Close the change notify channel on Shutdown (izouxv)
      • +
    • +
    • B2 +
        +
      • Fix "fatal error: concurrent map writes" (Nick Craig-Wood)
      • +
    • +
    • Iclouddrive +
        +
      • Add notes on ADP and Missing PCS cookies (Nick Craig-Wood)
      • +
    • +
    • Onedrive +
        +
      • Mark German (de) region as deprecated (Nick Craig-Wood)
      • +
    • +
    • S3 +
        +
      • Added new storage class to magalu provider (Bruno Fernandes)
      • +
      • Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe)
      • +
      • Add latest Linode Object Storage endpoints (jbagwell-akamai)
      • +
    • +

    v1.69.0 - 2025-01-12

    See commits

      diff --git a/MANUAL.md b/MANUAL.md index 8d2f405fef9f6..889d4fec9ddcf 100644 --- a/MANUAL.md +++ b/MANUAL.md @@ -1,7 +1,78 @@ % rclone(1) User Manual % Nick Craig-Wood -% Jan 12, 2025 +% Feb 14, 2025 +# NAME + +rclone - manage files on cloud storage + +# SYNOPSIS + +``` +Usage: + rclone [flags] + rclone [command] + +Available commands: + about Get quota information from the remote. + authorize Remote authorization. + backend Run a backend-specific command. + bisync Perform bidirectional synchronization between two paths. + cat Concatenates any files and sends them to stdout. + check Checks the files in the source and destination match. + checksum Checks the files in the destination against a SUM file. + cleanup Clean up the remote if possible. + completion Output completion script for a given shell. + config Enter an interactive configuration session. + copy Copy files from source to dest, skipping identical files. + copyto Copy files from source to dest, skipping identical files. + copyurl Copy the contents of the URL supplied content to dest:path. + cryptcheck Cryptcheck checks the integrity of an encrypted remote. + cryptdecode Cryptdecode returns unencrypted file names. + dedupe Interactively find duplicate filenames and delete/rename them. + delete Remove the files in path. + deletefile Remove a single file from remote. + gendocs Output markdown docs for rclone to the directory supplied. + gitannex Speaks with git-annex over stdin/stdout. + hashsum Produces a hashsum file for all the objects in the path. + help Show help for rclone commands, flags and backends. + link Generate public link to file/folder. + listremotes List all the remotes in the config file and defined in environment variables. + ls List the objects in the path with size and path. + lsd List all directories/containers/buckets in the path. + lsf List directories and objects in remote:path formatted for parsing. + lsjson List directories and objects in the path in JSON format. + lsl List the objects in path with modification time, size and path. + md5sum Produces an md5sum file for all the objects in the path. + mkdir Make the path if it doesn't already exist. + mount Mount the remote as file system on a mountpoint. + move Move files from source to dest. + moveto Move file or directory from source to dest. + ncdu Explore a remote with a text based user interface. + nfsmount Mount the remote as file system on a mountpoint. + obscure Obscure password for use in the rclone config file. + purge Remove the path and all of its contents. + rc Run a command against a running rclone. + rcat Copies standard input to file on remote. + rcd Run rclone listening to remote control commands only. + rmdir Remove the empty directory at path. + rmdirs Remove empty directories under the path. + selfupdate Update the rclone binary. + serve Serve a remote over a protocol. + settier Changes storage class/tier of objects in remote. + sha1sum Produces an sha1sum file for all the objects in the path. + size Prints the total size and number of objects in remote:path. + sync Make source and dest identical, modifying destination only. + test Run a test command + touch Create new file or change file modification time. + tree List the contents of the remote in a tree like fashion. + version Show the version number. + +Use "rclone [command] --help" for more information about a command. +Use "rclone help flags" for to see the global flags. +Use "rclone help backends" for a list of supported services. + +``` # Rclone syncs your files to cloud storage rclone logo @@ -1690,6 +1761,9 @@ include/exclude filters - everything will be removed. Use the delete files. To delete empty directories only, use command [rmdir](https://rclone.org/commands/rclone_rmdir/) or [rmdirs](https://rclone.org/commands/rclone_rmdirs/). +The concurrency of this operation is controlled by the `--checkers` global flag. However, some backends will +implement this command directly, in which case `--checkers` will be ignored. + **Important**: Since this can cause data loss, test first with the `--dry-run` or the `--interactive`/`-i` flag. @@ -3745,12 +3819,12 @@ password to re-encrypt the config. When `--password-command` is called to change the password then the environment variable `RCLONE_PASSWORD_CHANGE=1` will be set. So if -changing passwords programatically you can use the environment +changing passwords programmatically you can use the environment variable to distinguish which password you must supply. Alternatively you can remove the password first (with `rclone config encryption remove`), then set it again with this command which may be -easier if you don't mind the unecrypted config file being on the disk +easier if you don't mind the unencrypted config file being on the disk briefly. @@ -4290,7 +4364,7 @@ destination if there is one with the same name. Setting `--stdout` or making the output file name `-` will cause the output to be written to standard output. -## Troublshooting +## Troubleshooting If you can't get `rclone copyurl` to work then here are some things you can try: @@ -10581,7 +10655,7 @@ that it uses an on disk cache, but the cache entries are held as symlinks. Rclone will use the handle of the underlying file as the NFS handle which improves performance. This sort of cache can't be backed up and restored as the underlying handles will change. This is Linux -only. It requres running rclone as root or with `CAP_DAC_READ_SEARCH`. +only. It requires running rclone as root or with `CAP_DAC_READ_SEARCH`. You can run rclone with this extra permission by doing this to the rclone binary `sudo setcap cap_dac_read_search+ep /path/to/rclone`. @@ -11408,7 +11482,7 @@ secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false ``` -Note that setting `disable_multipart_uploads = true` is to work around +Note that setting `use_multipart_uploads = false` is to work around [a bug](#bugs) which will be fixed in due course. ## Bugs @@ -14444,6 +14518,11 @@ it to `false`. It is also possible to specify `--boolean=false` or parsed as `--boolean` and the `false` is parsed as an extra command line argument for rclone. +Options documented to take a `stringArray` parameter accept multiple +values. To pass more than one value, repeat the option; for example: +`--include value1 --include value2`. + + ### Time or duration options {#time-option} TIME or DURATION options can be specified as a duration string or a @@ -16755,7 +16834,7 @@ so they take exactly the same form. The options set by environment variables can be seen with the `-vv` flag, e.g. `rclone version -vv`. Options that can appear multiple times (type `stringArray`) are -treated slighly differently as environment variables can only be +treated slightly differently as environment variables can only be defined once. In order to allow a simple mechanism for adding one or many items, the input is treated as a [CSV encoded](https://godoc.org/encoding/csv) string. For example @@ -19937,7 +20016,7 @@ the `--vfs-cache-mode` is off, it will return an empty result. ], } -The `expiry` time is the time until the file is elegible for being +The `expiry` time is the time until the file is eligible for being uploaded in floating point seconds. This may go negative. As rclone only transfers `--transfers` files at once, only the lowest `--transfers` expiry times will have `uploading` as `true`. So there @@ -21018,7 +21097,7 @@ Flags for general networking and HTTP stuff. --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.1") ``` @@ -23066,7 +23145,7 @@ See the [bisync filters](#filtering) section and generic [--filter-from](https://rclone.org/filtering/#filter-from-read-filtering-patterns-from-a-file) documentation. An [example filters file](#example-filters-file) contains filters for -non-allowed files for synching with Dropbox. +non-allowed files for syncing with Dropbox. If you make changes to your filters file then bisync requires a run with `--resync`. This is a safety feature, which prevents existing files @@ -23243,7 +23322,7 @@ Using `--check-sync=false` will disable it and may significantly reduce the sync run times for very large numbers of files. The check may be run manually with `--check-sync=only`. It runs only the -integrity check and terminates without actually synching. +integrity check and terminates without actually syncing. Note that currently, `--check-sync` **only checks listing snapshots and NOT the actual files on the remotes.** Note also that the listing snapshots will not @@ -23720,7 +23799,7 @@ The `--include*`, `--exclude*`, and `--filter` flags are also supported. ### How to filter directories -Filtering portions of the directory tree is a critical feature for synching. +Filtering portions of the directory tree is a critical feature for syncing. Examples of directory trees (always beneath the Path1/Path2 root level) you may want to exclude from your sync: @@ -23829,7 +23908,7 @@ quashed by adding `--quiet` to the bisync command line. ## Example exclude-style filters files for use with Dropbox {#exclude-filters} -- Dropbox disallows synching the listed temporary and configuration/data files. +- Dropbox disallows syncing the listed temporary and configuration/data files. The `- ` filters exclude these files where ever they may occur in the sync tree. Consider adding similar exclusions for file types you don't need to sync, such as core dump and software build files. @@ -24163,7 +24242,7 @@ test command flags can be equally prefixed by a single `-` or double dash. - `go test . -case basic -remote local -remote2 local` runs the `test_basic` test case using only the local filesystem, - synching one local directory with another local directory. + syncing one local directory with another local directory. Test script output is to the console, while commands within scenario.txt have their output sent to the `.../workdir/test.log` file, which is finally compared to the golden copy. @@ -24394,6 +24473,9 @@ about _Unison_ and synchronization in general. ## Changelog +### `v1.69.1` +* Fixed an issue causing listings to not capture concurrent modifications under certain conditions + ### `v1.68` * Fixed an issue affecting backends that round modtimes to a lower precision. @@ -25680,7 +25762,7 @@ Notes on above: that `USER_NAME` has been created. 2. The Resource entry must include both resource ARNs, as one implies the bucket and the other implies the bucket's objects. -3. When using [s3-no-check-bucket](#s3-no-check-bucket) and the bucket already exsits, the `"arn:aws:s3:::BUCKET_NAME"` doesn't have to be included. +3. When using [s3-no-check-bucket](#s3-no-check-bucket) and the bucket already exists, the `"arn:aws:s3:::BUCKET_NAME"` doesn't have to be included. For reference, [here's an Ansible script](https://gist.github.com/ebridges/ebfc9042dd7c756cd101cfa807b7ae2b) that will generate one or more buckets that will work with `rclone sync`. @@ -28658,7 +28740,7 @@ location_constraint = au-nsw ### Rclone Serve S3 {#rclone} Rclone can serve any remote over the S3 protocol. For details see the -[rclone serve s3](https://rclone.org/commands/rclone_serve_http/) documentation. +[rclone serve s3](https://rclone.org/commands/rclone_serve_s3/) documentation. For example, to serve `remote:path` over s3, run the server like this: @@ -28678,8 +28760,8 @@ secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false ``` -Note that setting `disable_multipart_uploads = true` is to work around -[a bug](https://rclone.org/commands/rclone_serve_http/#bugs) which will be fixed in due course. +Note that setting `use_multipart_uploads = false` is to work around +[a bug](https://rclone.org/commands/rclone_serve_s3/#bugs) which will be fixed in due course. ### Scaleway @@ -29775,27 +29857,49 @@ Option endpoint. Endpoint for Linode Object Storage API. Choose a number from below, or type in your own value. Press Enter to leave empty. - 1 / Atlanta, GA (USA), us-southeast-1 + 1 / Amsterdam (Netherlands), nl-ams-1 + \ (nl-ams-1.linodeobjects.com) + 2 / Atlanta, GA (USA), us-southeast-1 \ (us-southeast-1.linodeobjects.com) - 2 / Chicago, IL (USA), us-ord-1 + 3 / Chennai (India), in-maa-1 + \ (in-maa-1.linodeobjects.com) + 4 / Chicago, IL (USA), us-ord-1 \ (us-ord-1.linodeobjects.com) - 3 / Frankfurt (Germany), eu-central-1 + 5 / Frankfurt (Germany), eu-central-1 \ (eu-central-1.linodeobjects.com) - 4 / Milan (Italy), it-mil-1 + 6 / Jakarta (Indonesia), id-cgk-1 + \ (id-cgk-1.linodeobjects.com) + 7 / London 2 (Great Britain), gb-lon-1 + \ (gb-lon-1.linodeobjects.com) + 8 / Los Angeles, CA (USA), us-lax-1 + \ (us-lax-1.linodeobjects.com) + 9 / Madrid (Spain), es-mad-1 + \ (es-mad-1.linodeobjects.com) +10 / Melbourne (Australia), au-mel-1 + \ (au-mel-1.linodeobjects.com) +11 / Miami, FL (USA), us-mia-1 + \ (us-mia-1.linodeobjects.com) +12 / Milan (Italy), it-mil-1 \ (it-mil-1.linodeobjects.com) - 5 / Newark, NJ (USA), us-east-1 +13 / Newark, NJ (USA), us-east-1 \ (us-east-1.linodeobjects.com) - 6 / Paris (France), fr-par-1 +14 / Osaka (Japan), jp-osa-1 + \ (jp-osa-1.linodeobjects.com) +15 / Paris (France), fr-par-1 \ (fr-par-1.linodeobjects.com) - 7 / Seattle, WA (USA), us-sea-1 +16 / São Paulo (Brazil), br-gru-1 + \ (br-gru-1.linodeobjects.com) +17 / Seattle, WA (USA), us-sea-1 \ (us-sea-1.linodeobjects.com) - 8 / Singapore ap-south-1 +18 / Singapore, ap-south-1 \ (ap-south-1.linodeobjects.com) - 9 / Stockholm (Sweden), se-sto-1 +19 / Singapore 2, sg-sin-1 + \ (sg-sin-1.linodeobjects.com) +20 / Stockholm (Sweden), se-sto-1 \ (se-sto-1.linodeobjects.com) -10 / Washington, DC, (USA), us-iad-1 +21 / Washington, DC, (USA), us-iad-1 \ (us-iad-1.linodeobjects.com) -endpoint> 3 +endpoint> 5 Option acl. Canned ACL used when creating buckets and storing or copying objects. @@ -34415,7 +34519,7 @@ strong random number generator. The nonce is incremented for each chunk read making sure each nonce is unique for each block written. The chance of a nonce being reused is minuscule. If you wrote an exabyte of data (10¹⁸ bytes) you would have a probability of -approximately 2×10⁻³² of re-using a nonce. +approximately 2×10⁻³² of reusing a nonce. #### Chunk @@ -41561,7 +41665,7 @@ Enter a value. config_2fa> 2FACODE Remote config -------------------- -[koofr] +[iclouddrive] - type: iclouddrive - apple_id: APPLEID - password: *** ENCRYPTED *** @@ -41578,6 +41682,20 @@ y/e/d> y ADP is currently unsupported and need to be disabled +On iPhone, Settings `>` Apple Account `>` iCloud `>` 'Access iCloud Data on the Web' must be ON, and 'Advanced Data Protection' OFF. + +## Troubleshooting + +### Missing PCS cookies from the request + +This means you have Advanced Data Protection (ADP) turned on. This is not supported at the moment. If you want to use rclone you will have to turn it off. See above for how to turn it off. + +You will need to clear the `cookies` and the `trust_token` fields in the config. Or you can delete the remote config and start again. + +You should then run `rclone reconnect remote:`. + +Note that changing the ADP setting may not take effect immediately - you may need to wait a few hours or a day before you can get rclone to work - keep clearing the config entry and running `rclone reconnect remote:` until rclone functions properly. + ### Standard options @@ -46035,7 +46153,7 @@ Properties: - "us" - Microsoft Cloud for US Government - "de" - - Microsoft Cloud Germany + - Microsoft Cloud Germany (deprecated - try global region first). - "cn" - Azure and Office 365 operated by Vnet Group in China @@ -46652,6 +46770,28 @@ See the [metadata](https://rclone.org/docs/#metadata) docs for more info. +### Impersonate other users as Admin + +Unlike Google Drive and impersonating any domain user via service accounts, OneDrive requires you to authenticate as an admin account, and manually setup a remote per user you wish to impersonate. + +1. In [Microsoft 365 Admin Center](https://admin.microsoft.com), open each user you need to "impersonate" and go to the OneDrive section. There is a heading called "Get access to files", you need to click to create the link, this creates the link of the format `https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/` but also changes the permissions so you your admin user has access. +2. Then in powershell run the following commands: +```console +Install-Module Microsoft.Graph -Scope CurrentUser -Repository PSGallery -Force +Import-Module Microsoft.Graph.Files +Connect-MgGraph -Scopes "Files.ReadWrite.All" +# Follow the steps to allow access to your admin user +# Then run this for each user you want to impersonate to get the Drive ID +Get-MgUserDefaultDrive -UserId '{emailaddress}' +# This will give you output of the format: +# Name Id DriveType CreatedDateTime +# ---- -- --------- --------------- +# OneDrive b!XYZ123 business 14/10/2023 1:00:58 pm + +``` +3. Then in rclone add a onedrive remote type, and use the `Type in driveID` with the DriveID you got in the previous step. One remote per user. It will then confirm the drive ID, and hopefully give you a message of `Found drive "root" of type "business"` and then include the URL of the format `https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/Documents` + + ## Limitations If you don't use rclone for 90 days the refresh token will @@ -56509,6 +56649,32 @@ Options: # Changelog +## v1.69.1 - 2025-02-14 + +[See commits](https://github.com/rclone/rclone/compare/v1.69.0...v1.69.1) + +* Bug Fixes + * lib/oauthutil: Fix redirect URL mismatch errors (Nick Craig-Wood) + * bisync: Fix listings missing concurrent modifications (nielash) + * serve s3: Fix list objects encoding-type (Nick Craig-Wood) + * fs: Fix confusing "didn't find section in config file" error (Nick Craig-Wood) + * doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt Ickstadt, Nick Craig-Wood, Tim White, Zachary Vorhies) + * build: Added parallel docker builds and caching for go build in the container (Anagh Kumar Baranwal) +* VFS + * Fix the cache failing to upload symlinks when `--links` was specified (Nick Craig-Wood) + * Fix race detected by race detector (Nick Craig-Wood) + * Close the change notify channel on Shutdown (izouxv) +* B2 + * Fix "fatal error: concurrent map writes" (Nick Craig-Wood) +* Iclouddrive + * Add notes on ADP and Missing PCS cookies (Nick Craig-Wood) +* Onedrive + * Mark German (de) region as deprecated (Nick Craig-Wood) +* S3 + * Added new storage class to magalu provider (Bruno Fernandes) + * Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe) + * Add latest Linode Object Storage endpoints (jbagwell-akamai) + ## v1.69.0 - 2025-01-12 [See commits](https://github.com/rclone/rclone/compare/v1.68.0...v1.69.0) diff --git a/MANUAL.txt b/MANUAL.txt index 9247df8dce283..ea2b464f91c23 100644 --- a/MANUAL.txt +++ b/MANUAL.txt @@ -1,6 +1,75 @@ rclone(1) User Manual Nick Craig-Wood -Jan 12, 2025 +Feb 14, 2025 + +NAME + +rclone - manage files on cloud storage + +SYNOPSIS + + Usage: + rclone [flags] + rclone [command] + + Available commands: + about Get quota information from the remote. + authorize Remote authorization. + backend Run a backend-specific command. + bisync Perform bidirectional synchronization between two paths. + cat Concatenates any files and sends them to stdout. + check Checks the files in the source and destination match. + checksum Checks the files in the destination against a SUM file. + cleanup Clean up the remote if possible. + completion Output completion script for a given shell. + config Enter an interactive configuration session. + copy Copy files from source to dest, skipping identical files. + copyto Copy files from source to dest, skipping identical files. + copyurl Copy the contents of the URL supplied content to dest:path. + cryptcheck Cryptcheck checks the integrity of an encrypted remote. + cryptdecode Cryptdecode returns unencrypted file names. + dedupe Interactively find duplicate filenames and delete/rename them. + delete Remove the files in path. + deletefile Remove a single file from remote. + gendocs Output markdown docs for rclone to the directory supplied. + gitannex Speaks with git-annex over stdin/stdout. + hashsum Produces a hashsum file for all the objects in the path. + help Show help for rclone commands, flags and backends. + link Generate public link to file/folder. + listremotes List all the remotes in the config file and defined in environment variables. + ls List the objects in the path with size and path. + lsd List all directories/containers/buckets in the path. + lsf List directories and objects in remote:path formatted for parsing. + lsjson List directories and objects in the path in JSON format. + lsl List the objects in path with modification time, size and path. + md5sum Produces an md5sum file for all the objects in the path. + mkdir Make the path if it doesn't already exist. + mount Mount the remote as file system on a mountpoint. + move Move files from source to dest. + moveto Move file or directory from source to dest. + ncdu Explore a remote with a text based user interface. + nfsmount Mount the remote as file system on a mountpoint. + obscure Obscure password for use in the rclone config file. + purge Remove the path and all of its contents. + rc Run a command against a running rclone. + rcat Copies standard input to file on remote. + rcd Run rclone listening to remote control commands only. + rmdir Remove the empty directory at path. + rmdirs Remove empty directories under the path. + selfupdate Update the rclone binary. + serve Serve a remote over a protocol. + settier Changes storage class/tier of objects in remote. + sha1sum Produces an sha1sum file for all the objects in the path. + size Prints the total size and number of objects in remote:path. + sync Make source and dest identical, modifying destination only. + test Run a test command + touch Create new file or change file modification time. + tree List the contents of the remote in a tree like fashion. + version Show the version number. + + Use "rclone [command] --help" for more information about a command. + Use "rclone help flags" for to see the global flags. + Use "rclone help backends" for a list of supported services. Rclone syncs your files to cloud storage @@ -1600,6 +1669,10 @@ include/exclude filters - everything will be removed. Use the delete command if you want to selectively delete files. To delete empty directories only, use command rmdir or rmdirs. +The concurrency of this operation is controlled by the --checkers global +flag. However, some backends will implement this command directly, in +which case --checkers will be ignored. + Important: Since this can cause data loss, test first with the --dry-run or the --interactive/-i flag. @@ -3467,12 +3540,12 @@ re-encrypt the config. When --password-command is called to change the password then the environment variable RCLONE_PASSWORD_CHANGE=1 will be set. So if -changing passwords programatically you can use the environment variable +changing passwords programmatically you can use the environment variable to distinguish which password you must supply. Alternatively you can remove the password first (with rclone config encryption remove), then set it again with this command -which may be easier if you don't mind the unecrypted config file being +which may be easier if you don't mind the unencrypted config file being on the disk briefly. rclone config encryption set [flags] @@ -3949,7 +4022,7 @@ there is one with the same name. Setting --stdout or making the output file name - will cause the output to be written to standard output. -Troublshooting +Troubleshooting If you can't get rclone copyurl to work then here are some things you can try: @@ -10102,7 +10175,7 @@ uses an on disk cache, but the cache entries are held as symlinks. Rclone will use the handle of the underlying file as the NFS handle which improves performance. This sort of cache can't be backed up and restored as the underlying handles will change. This is Linux only. It -requres running rclone as root or with CAP_DAC_READ_SEARCH. You can run +requires running rclone as root or with CAP_DAC_READ_SEARCH. You can run rclone with this extra permission by doing this to the rclone binary sudo setcap cap_dac_read_search+ep /path/to/rclone. @@ -10903,8 +10976,8 @@ which is defined like this: secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false -Note that setting disable_multipart_uploads = true is to work around a -bug which will be fixed in due course. +Note that setting use_multipart_uploads = false is to work around a bug +which will be fixed in due course. Bugs @@ -13895,6 +13968,10 @@ also possible to specify --boolean=false or --boolean=true. Note that --boolean false is not valid - this is parsed as --boolean and the false is parsed as an extra command line argument for rclone. +Options documented to take a stringArray parameter accept multiple +values. To pass more than one value, repeat the option; for example: +--include value1 --include value2. + Time or duration options TIME or DURATION options can be specified as a duration string or a time @@ -16177,7 +16254,7 @@ The options set by environment variables can be seen with the -vv flag, e.g. rclone version -vv. Options that can appear multiple times (type stringArray) are treated -slighly differently as environment variables can only be defined once. +slightly differently as environment variables can only be defined once. In order to allow a simple mechanism for adding one or many items, the input is treated as a CSV encoded string. For example @@ -19420,7 +19497,7 @@ This is only useful if --vfs-cache-mode > off. If you call it when the ], } -The expiry time is the time until the file is elegible for being +The expiry time is the time until the file is eligible for being uploaded in floating point seconds. This may go negative. As rclone only transfers --transfers files at once, only the lowest --transfers expiry times will have uploading as true. So there may be files with negative @@ -20569,7 +20646,7 @@ Flags for general networking and HTTP stuff. --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.1") Performance @@ -22531,7 +22608,7 @@ Also see the all files changed check. By using rclone filter features you can exclude file types or directory sub-trees from the sync. See the bisync filters section and generic --filter-from documentation. An example filters file contains filters -for non-allowed files for synching with Dropbox. +for non-allowed files for syncing with Dropbox. If you make changes to your filters file then bisync requires a run with --resync. This is a safety feature, which prevents existing files on the @@ -22704,7 +22781,7 @@ of a sync. Using --check-sync=false will disable it and may significantly reduce the sync run times for very large numbers of files. The check may be run manually with --check-sync=only. It runs only the -integrity check and terminates without actually synching. +integrity check and terminates without actually syncing. Note that currently, --check-sync only checks listing snapshots and NOT the actual files on the remotes. Note also that the listing snapshots @@ -23237,7 +23314,7 @@ supported. How to filter directories Filtering portions of the directory tree is a critical feature for -synching. +syncing. Examples of directory trees (always beneath the Path1/Path2 root level) you may want to exclude from your sync: - Directory trees containing @@ -23348,7 +23425,7 @@ This noise can be quashed by adding --quiet to the bisync command line. Example exclude-style filters files for use with Dropbox -- Dropbox disallows synching the listed temporary and +- Dropbox disallows syncing the listed temporary and configuration/data files. The `- ` filters exclude these files where ever they may occur in the sync tree. Consider adding similar exclusions for file types you don't need to sync, such as core dump @@ -23668,7 +23745,7 @@ dash. Running tests - go test . -case basic -remote local -remote2 local runs the - test_basic test case using only the local filesystem, synching one + test_basic test case using only the local filesystem, syncing one local directory with another local directory. Test script output is to the console, while commands within scenario.txt have their output sent to the .../workdir/test.log file, which is finally compared to @@ -23901,6 +23978,11 @@ Unison and synchronization in general. Changelog +v1.69.1 + +- Fixed an issue causing listings to not capture concurrent + modifications under certain conditions + v1.68 - Fixed an issue affecting backends that round modtimes to a lower @@ -25192,7 +25274,7 @@ Notes on above: that USER_NAME has been created. 2. The Resource entry must include both resource ARNs, as one implies the bucket and the other implies the bucket's objects. -3. When using s3-no-check-bucket and the bucket already exsits, the +3. When using s3-no-check-bucket and the bucket already exists, the "arn:aws:s3:::BUCKET_NAME" doesn't have to be included. For reference, here's an Ansible script that will generate one or more @@ -28155,8 +28237,8 @@ this: secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false -Note that setting disable_multipart_uploads = true is to work around a -bug which will be fixed in due course. +Note that setting use_multipart_uploads = false is to work around a bug +which will be fixed in due course. Scaleway @@ -29203,27 +29285,49 @@ This will guide you through an interactive setup process. Endpoint for Linode Object Storage API. Choose a number from below, or type in your own value. Press Enter to leave empty. - 1 / Atlanta, GA (USA), us-southeast-1 + 1 / Amsterdam (Netherlands), nl-ams-1 + \ (nl-ams-1.linodeobjects.com) + 2 / Atlanta, GA (USA), us-southeast-1 \ (us-southeast-1.linodeobjects.com) - 2 / Chicago, IL (USA), us-ord-1 + 3 / Chennai (India), in-maa-1 + \ (in-maa-1.linodeobjects.com) + 4 / Chicago, IL (USA), us-ord-1 \ (us-ord-1.linodeobjects.com) - 3 / Frankfurt (Germany), eu-central-1 + 5 / Frankfurt (Germany), eu-central-1 \ (eu-central-1.linodeobjects.com) - 4 / Milan (Italy), it-mil-1 + 6 / Jakarta (Indonesia), id-cgk-1 + \ (id-cgk-1.linodeobjects.com) + 7 / London 2 (Great Britain), gb-lon-1 + \ (gb-lon-1.linodeobjects.com) + 8 / Los Angeles, CA (USA), us-lax-1 + \ (us-lax-1.linodeobjects.com) + 9 / Madrid (Spain), es-mad-1 + \ (es-mad-1.linodeobjects.com) + 10 / Melbourne (Australia), au-mel-1 + \ (au-mel-1.linodeobjects.com) + 11 / Miami, FL (USA), us-mia-1 + \ (us-mia-1.linodeobjects.com) + 12 / Milan (Italy), it-mil-1 \ (it-mil-1.linodeobjects.com) - 5 / Newark, NJ (USA), us-east-1 + 13 / Newark, NJ (USA), us-east-1 \ (us-east-1.linodeobjects.com) - 6 / Paris (France), fr-par-1 + 14 / Osaka (Japan), jp-osa-1 + \ (jp-osa-1.linodeobjects.com) + 15 / Paris (France), fr-par-1 \ (fr-par-1.linodeobjects.com) - 7 / Seattle, WA (USA), us-sea-1 + 16 / São Paulo (Brazil), br-gru-1 + \ (br-gru-1.linodeobjects.com) + 17 / Seattle, WA (USA), us-sea-1 \ (us-sea-1.linodeobjects.com) - 8 / Singapore ap-south-1 + 18 / Singapore, ap-south-1 \ (ap-south-1.linodeobjects.com) - 9 / Stockholm (Sweden), se-sto-1 + 19 / Singapore 2, sg-sin-1 + \ (sg-sin-1.linodeobjects.com) + 20 / Stockholm (Sweden), se-sto-1 \ (se-sto-1.linodeobjects.com) - 10 / Washington, DC, (USA), us-iad-1 + 21 / Washington, DC, (USA), us-iad-1 \ (us-iad-1.linodeobjects.com) - endpoint> 3 + endpoint> 5 Option acl. Canned ACL used when creating buckets and storing or copying objects. @@ -33757,7 +33861,7 @@ The initial nonce is generated from the operating systems crypto strong random number generator. The nonce is incremented for each chunk read making sure each nonce is unique for each block written. The chance of a nonce being reused is minuscule. If you wrote an exabyte of data (10¹⁸ -bytes) you would have a probability of approximately 2×10⁻³² of re-using +bytes) you would have a probability of approximately 2×10⁻³² of reusing a nonce. Chunk @@ -40978,7 +41082,7 @@ This will guide you through an interactive setup process: config_2fa> 2FACODE Remote config -------------------- - [koofr] + [iclouddrive] - type: iclouddrive - apple_id: APPLEID - password: *** ENCRYPTED *** @@ -40994,6 +41098,27 @@ Advanced Data Protection ADP is currently unsupported and need to be disabled +On iPhone, Settings > Apple Account > iCloud > 'Access iCloud Data on +the Web' must be ON, and 'Advanced Data Protection' OFF. + +Troubleshooting + +Missing PCS cookies from the request + +This means you have Advanced Data Protection (ADP) turned on. This is +not supported at the moment. If you want to use rclone you will have to +turn it off. See above for how to turn it off. + +You will need to clear the cookies and the trust_token fields in the +config. Or you can delete the remote config and start again. + +You should then run rclone reconnect remote:. + +Note that changing the ADP setting may not take effect immediately - you +may need to wait a few hours or a day before you can get rclone to work +- keep clearing the config entry and running rclone reconnect remote: +until rclone functions properly. + Standard options Here are the Standard options specific to iclouddrive (iCloud Drive). @@ -45589,7 +45714,8 @@ Properties: - "us" - Microsoft Cloud for US Government - "de" - - Microsoft Cloud Germany + - Microsoft Cloud Germany (deprecated - try global region + first). - "cn" - Azure and Office 365 operated by Vnet Group in China @@ -46248,6 +46374,38 @@ Here are the possible system metadata items for the onedrive backend. See the metadata docs for more info. +Impersonate other users as Admin + +Unlike Google Drive and impersonating any domain user via service +accounts, OneDrive requires you to authenticate as an admin account, and +manually setup a remote per user you wish to impersonate. + +1. In Microsoft 365 Admin Center, open each user you need to + "impersonate" and go to the OneDrive section. There is a heading + called "Get access to files", you need to click to create the link, + this creates the link of the format + https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/ + but also changes the permissions so you your admin user has access. +2. Then in powershell run the following commands: + + Install-Module Microsoft.Graph -Scope CurrentUser -Repository PSGallery -Force + Import-Module Microsoft.Graph.Files + Connect-MgGraph -Scopes "Files.ReadWrite.All" + # Follow the steps to allow access to your admin user + # Then run this for each user you want to impersonate to get the Drive ID + Get-MgUserDefaultDrive -UserId '{emailaddress}' + # This will give you output of the format: + # Name Id DriveType CreatedDateTime + # ---- -- --------- --------------- + # OneDrive b!XYZ123 business 14/10/2023 1:00:58 pm + +3. Then in rclone add a onedrive remote type, and use the + Type in driveID with the DriveID you got in the previous step. One + remote per user. It will then confirm the drive ID, and hopefully + give you a message of Found drive "root" of type "business" and then + include the URL of the format + https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/Documents + Limitations If you don't use rclone for 90 days the refresh token will expire. This @@ -56157,6 +56315,37 @@ Options: Changelog +v1.69.1 - 2025-02-14 + +See commits + +- Bug Fixes + - lib/oauthutil: Fix redirect URL mismatch errors (Nick + Craig-Wood) + - bisync: Fix listings missing concurrent modifications (nielash) + - serve s3: Fix list objects encoding-type (Nick Craig-Wood) + - fs: Fix confusing "didn't find section in config file" error + (Nick Craig-Wood) + - doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt + Ickstadt, Nick Craig-Wood, Tim White, Zachary Vorhies) + - build: Added parallel docker builds and caching for go build in + the container (Anagh Kumar Baranwal) +- VFS + - Fix the cache failing to upload symlinks when --links was + specified (Nick Craig-Wood) + - Fix race detected by race detector (Nick Craig-Wood) + - Close the change notify channel on Shutdown (izouxv) +- B2 + - Fix "fatal error: concurrent map writes" (Nick Craig-Wood) +- Iclouddrive + - Add notes on ADP and Missing PCS cookies (Nick Craig-Wood) +- Onedrive + - Mark German (de) region as deprecated (Nick Craig-Wood) +- S3 + - Added new storage class to magalu provider (Bruno Fernandes) + - Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe) + - Add latest Linode Object Storage endpoints (jbagwell-akamai) + v1.69.0 - 2025-01-12 See commits diff --git a/docs/content/changelog.md b/docs/content/changelog.md index e3170bbf7dbfc..0bdf7c317c0fc 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -5,6 +5,32 @@ description: "Rclone Changelog" # Changelog +## v1.69.1 - 2025-02-14 + +[See commits](https://github.com/rclone/rclone/compare/v1.69.0...v1.69.1) + +* Bug Fixes + * lib/oauthutil: Fix redirect URL mismatch errors (Nick Craig-Wood) + * bisync: Fix listings missing concurrent modifications (nielash) + * serve s3: Fix list objects encoding-type (Nick Craig-Wood) + * fs: Fix confusing "didn't find section in config file" error (Nick Craig-Wood) + * doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt Ickstadt, Nick Craig-Wood, Tim White, Zachary Vorhies) + * build: Added parallel docker builds and caching for go build in the container (Anagh Kumar Baranwal) +* VFS + * Fix the cache failing to upload symlinks when `--links` was specified (Nick Craig-Wood) + * Fix race detected by race detector (Nick Craig-Wood) + * Close the change notify channel on Shutdown (izouxv) +* B2 + * Fix "fatal error: concurrent map writes" (Nick Craig-Wood) +* Iclouddrive + * Add notes on ADP and Missing PCS cookies (Nick Craig-Wood) +* Onedrive + * Mark German (de) region as deprecated (Nick Craig-Wood) +* S3 + * Added new storage class to magalu provider (Bruno Fernandes) + * Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe) + * Add latest Linode Object Storage endpoints (jbagwell-akamai) + ## v1.69.0 - 2025-01-12 [See commits](https://github.com/rclone/rclone/compare/v1.68.0...v1.69.0) diff --git a/docs/content/commands/rclone.md b/docs/content/commands/rclone.md index 0921995de23a7..3890f0b9164ef 100644 --- a/docs/content/commands/rclone.md +++ b/docs/content/commands/rclone.md @@ -965,7 +965,7 @@ rclone [flags] --use-json-log Use json log format --use-mmap Use mmap allocator (see docs) --use-server-modtime Use server modified time instead of object metadata - --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.1") -v, --verbose count Print lots more stuff (repeat for more) -V, --version Print the version number --webdav-auth-redirect Preserve authentication on redirect diff --git a/docs/content/commands/rclone_config_encryption_set.md b/docs/content/commands/rclone_config_encryption_set.md index b02dff900bd3d..780c086dc0b0b 100644 --- a/docs/content/commands/rclone_config_encryption_set.md +++ b/docs/content/commands/rclone_config_encryption_set.md @@ -21,12 +21,12 @@ password to re-encrypt the config. When `--password-command` is called to change the password then the environment variable `RCLONE_PASSWORD_CHANGE=1` will be set. So if -changing passwords programatically you can use the environment +changing passwords programmatically you can use the environment variable to distinguish which password you must supply. Alternatively you can remove the password first (with `rclone config encryption remove`), then set it again with this command which may be -easier if you don't mind the unecrypted config file being on the disk +easier if you don't mind the unencrypted config file being on the disk briefly. diff --git a/docs/content/commands/rclone_copyurl.md b/docs/content/commands/rclone_copyurl.md index 061644fa39368..2bdccf5fd5d64 100644 --- a/docs/content/commands/rclone_copyurl.md +++ b/docs/content/commands/rclone_copyurl.md @@ -28,7 +28,7 @@ destination if there is one with the same name. Setting `--stdout` or making the output file name `-` will cause the output to be written to standard output. -## Troublshooting +## Troubleshooting If you can't get `rclone copyurl` to work then here are some things you can try: diff --git a/docs/content/commands/rclone_purge.md b/docs/content/commands/rclone_purge.md index ab191f57e94a4..c1fded41c4eec 100644 --- a/docs/content/commands/rclone_purge.md +++ b/docs/content/commands/rclone_purge.md @@ -15,6 +15,9 @@ include/exclude filters - everything will be removed. Use the delete files. To delete empty directories only, use command [rmdir](/commands/rclone_rmdir/) or [rmdirs](/commands/rclone_rmdirs/). +The concurrency of this operation is controlled by the `--checkers` global flag. However, some backends will +implement this command directly, in which case `--checkers` will be ignored. + **Important**: Since this can cause data loss, test first with the `--dry-run` or the `--interactive`/`-i` flag. diff --git a/docs/content/commands/rclone_serve_nfs.md b/docs/content/commands/rclone_serve_nfs.md index ed8358ec24472..39deb36a933b7 100644 --- a/docs/content/commands/rclone_serve_nfs.md +++ b/docs/content/commands/rclone_serve_nfs.md @@ -7,8 +7,6 @@ versionIntroduced: v1.65 --- # rclone serve nfs -*Not available in Windows.* - Serve the remote as an NFS mount ## Synopsis @@ -55,7 +53,7 @@ that it uses an on disk cache, but the cache entries are held as symlinks. Rclone will use the handle of the underlying file as the NFS handle which improves performance. This sort of cache can't be backed up and restored as the underlying handles will change. This is Linux -only. It requres running rclone as root or with `CAP_DAC_READ_SEARCH`. +only. It requires running rclone as root or with `CAP_DAC_READ_SEARCH`. You can run rclone with this extra permission by doing this to the rclone binary `sudo setcap cap_dac_read_search+ep /path/to/rclone`. diff --git a/docs/content/commands/rclone_serve_s3.md b/docs/content/commands/rclone_serve_s3.md index 40813321b36de..8785d20659908 100644 --- a/docs/content/commands/rclone_serve_s3.md +++ b/docs/content/commands/rclone_serve_s3.md @@ -82,7 +82,7 @@ secret_access_key = SECRET_ACCESS_KEY use_multipart_uploads = false ``` -Note that setting `disable_multipart_uploads = true` is to work around +Note that setting `use_multipart_uploads = false` is to work around [a bug](#bugs) which will be fixed in due course. ## Bugs diff --git a/docs/content/flags.md b/docs/content/flags.md index f9d32c490d527..e022ed0ec43e0 100644 --- a/docs/content/flags.md +++ b/docs/content/flags.md @@ -9,8 +9,6 @@ description: "Rclone Global Flags" This describes the global flags available to every rclone command split into groups. -See the [Options section](/docs/#options) for syntax and usage advice. - ## Copy @@ -118,7 +116,7 @@ Flags for general networking and HTTP stuff. --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.0") + --user-agent string Set the user-agent to a specified string (default "rclone/v1.69.1") ``` diff --git a/docs/content/googlephotos.md b/docs/content/googlephotos.md index c3549d1589053..26faedcc6b7f1 100644 --- a/docs/content/googlephotos.md +++ b/docs/content/googlephotos.md @@ -384,7 +384,7 @@ Use the gphotosdl proxy for downloading the full resolution images The Google API will deliver images and video which aren't full resolution, and/or have EXIF data missing. -However if you use the gphotosdl proxy then you can download original, +However if you ue the gphotosdl proxy tnen you can download original, unchanged images. This runs a headless browser in the background. diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index 91c8520831d1a..1927123baff05 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -319,7 +319,7 @@ Properties: - "us" - Microsoft Cloud for US Government - "de" - - Microsoft Cloud Germany + - Microsoft Cloud Germany (deprecated - try global region first). - "cn" - Azure and Office 365 operated by Vnet Group in China diff --git a/rclone.1 b/rclone.1 index 28e0c3304b860..57bda9b82f957 100644 --- a/rclone.1 +++ b/rclone.1 @@ -1,8 +1,79 @@ .\"t .\" Automatically generated by Pandoc 2.9.2.1 .\" -.TH "rclone" "1" "Jan 12, 2025" "User Manual" "" +.TH "rclone" "1" "Feb 14, 2025" "User Manual" "" .hy +.SH NAME +.PP +rclone - manage files on cloud storage +.SH SYNOPSIS +.IP +.nf +\f[C] +Usage: + rclone [flags] + rclone [command] + +Available commands: + about Get quota information from the remote. + authorize Remote authorization. + backend Run a backend-specific command. + bisync Perform bidirectional synchronization between two paths. + cat Concatenates any files and sends them to stdout. + check Checks the files in the source and destination match. + checksum Checks the files in the destination against a SUM file. + cleanup Clean up the remote if possible. + completion Output completion script for a given shell. + config Enter an interactive configuration session. + copy Copy files from source to dest, skipping identical files. + copyto Copy files from source to dest, skipping identical files. + copyurl Copy the contents of the URL supplied content to dest:path. + cryptcheck Cryptcheck checks the integrity of an encrypted remote. + cryptdecode Cryptdecode returns unencrypted file names. + dedupe Interactively find duplicate filenames and delete/rename them. + delete Remove the files in path. + deletefile Remove a single file from remote. + gendocs Output markdown docs for rclone to the directory supplied. + gitannex Speaks with git-annex over stdin/stdout. + hashsum Produces a hashsum file for all the objects in the path. + help Show help for rclone commands, flags and backends. + link Generate public link to file/folder. + listremotes List all the remotes in the config file and defined in environment variables. + ls List the objects in the path with size and path. + lsd List all directories/containers/buckets in the path. + lsf List directories and objects in remote:path formatted for parsing. + lsjson List directories and objects in the path in JSON format. + lsl List the objects in path with modification time, size and path. + md5sum Produces an md5sum file for all the objects in the path. + mkdir Make the path if it doesn\[aq]t already exist. + mount Mount the remote as file system on a mountpoint. + move Move files from source to dest. + moveto Move file or directory from source to dest. + ncdu Explore a remote with a text based user interface. + nfsmount Mount the remote as file system on a mountpoint. + obscure Obscure password for use in the rclone config file. + purge Remove the path and all of its contents. + rc Run a command against a running rclone. + rcat Copies standard input to file on remote. + rcd Run rclone listening to remote control commands only. + rmdir Remove the empty directory at path. + rmdirs Remove empty directories under the path. + selfupdate Update the rclone binary. + serve Serve a remote over a protocol. + settier Changes storage class/tier of objects in remote. + sha1sum Produces an sha1sum file for all the objects in the path. + size Prints the total size and number of objects in remote:path. + sync Make source and dest identical, modifying destination only. + test Run a test command + touch Create new file or change file modification time. + tree List the contents of the remote in a tree like fashion. + version Show the version number. + +Use \[dq]rclone [command] --help\[dq] for more information about a command. +Use \[dq]rclone help flags\[dq] for to see the global flags. +Use \[dq]rclone help backends\[dq] for a list of supported services. +\f[R] +.fi .SH Rclone syncs your files to cloud storage .PP .IP \[bu] 2 @@ -2238,6 +2309,11 @@ To delete empty directories only, use command rmdir (https://rclone.org/commands/rclone_rmdir/) or rmdirs (https://rclone.org/commands/rclone_rmdirs/). .PP +The concurrency of this operation is controlled by the +\f[C]--checkers\f[R] global flag. +However, some backends will implement this command directly, in which +case \f[C]--checkers\f[R] will be ignored. +.PP \f[B]Important\f[R]: Since this can cause data loss, test first with the \f[C]--dry-run\f[R] or the \f[C]--interactive\f[R]/\f[C]-i\f[R] flag. .IP @@ -4652,12 +4728,12 @@ password to re-encrypt the config. .PP When \f[C]--password-command\f[R] is called to change the password then the environment variable \f[C]RCLONE_PASSWORD_CHANGE=1\f[R] will be set. -So if changing passwords programatically you can use the environment +So if changing passwords programmatically you can use the environment variable to distinguish which password you must supply. .PP Alternatively you can remove the password first (with \f[C]rclone config encryption remove\f[R]), then set it again with this -command which may be easier if you don\[aq]t mind the unecrypted config +command which may be easier if you don\[aq]t mind the unencrypted config file being on the disk briefly. .IP .nf @@ -5273,7 +5349,7 @@ destination if there is one with the same name. .PP Setting \f[C]--stdout\f[R] or making the output file name \f[C]-\f[R] will cause the output to be written to standard output. -.SS Troublshooting +.SS Troubleshooting .PP If you can\[aq]t get \f[C]rclone copyurl\f[R] to work then here are some things you can try: @@ -12993,7 +13069,8 @@ which improves performance. This sort of cache can\[aq]t be backed up and restored as the underlying handles will change. This is Linux only. -It requres running rclone as root or with \f[C]CAP_DAC_READ_SEARCH\f[R]. +It requires running rclone as root or with +\f[C]CAP_DAC_READ_SEARCH\f[R]. You can run rclone with this extra permission by doing this to the rclone binary \f[C]sudo setcap cap_dac_read_search+ep /path/to/rclone\f[R]. @@ -13973,7 +14050,7 @@ use_multipart_uploads = false \f[R] .fi .PP -Note that setting \f[C]disable_multipart_uploads = true\f[R] is to work +Note that setting \f[C]use_multipart_uploads = false\f[R] is to work around a bug which will be fixed in due course. .SS Bugs .PP @@ -17806,6 +17883,11 @@ It is also possible to specify \f[C]--boolean=false\f[R] or Note that \f[C]--boolean false\f[R] is not valid - this is parsed as \f[C]--boolean\f[R] and the \f[C]false\f[R] is parsed as an extra command line argument for rclone. +.PP +Options documented to take a \f[C]stringArray\f[R] parameter accept +multiple values. +To pass more than one value, repeat the option; for example: +\f[C]--include value1 --include value2\f[R]. .SS Time or duration options .PP TIME or DURATION options can be specified as a duration string or a time @@ -20455,8 +20537,8 @@ The options set by environment variables can be seen with the \f[C]rclone version -vv\f[R]. .PP Options that can appear multiple times (type \f[C]stringArray\f[R]) are -treated slighly differently as environment variables can only be defined -once. +treated slightly differently as environment variables can only be +defined once. In order to allow a simple mechanism for adding one or many items, the input is treated as a CSV encoded (https://godoc.org/encoding/csv) string. @@ -24731,7 +24813,7 @@ return an empty result. \f[R] .fi .PP -The \f[C]expiry\f[R] time is the time until the file is elegible for +The \f[C]expiry\f[R] time is the time until the file is eligible for being uploaded in floating point seconds. This may go negative. As rclone only transfers \f[C]--transfers\f[R] files at once, only the @@ -28442,7 +28524,7 @@ Flags for general networking and HTTP stuff. --tpslimit float Limit HTTP transactions per second to this --tpslimit-burst int Max burst of transactions for --tpslimit (default 1) --use-cookies Enable session cookiejar - --user-agent string Set the user-agent to a specified string (default \[dq]rclone/v1.69.0\[dq]) + --user-agent string Set the user-agent to a specified string (default \[dq]rclone/v1.69.1\[dq]) \f[R] .fi .SS Performance @@ -30761,7 +30843,7 @@ See the bisync filters section and generic --filter-from (https://rclone.org/filtering/#filter-from-read-filtering-patterns-from-a-file) documentation. An example filters file contains filters for non-allowed files for -synching with Dropbox. +syncing with Dropbox. .PP If you make changes to your filters file then bisync requires a run with \f[C]--resync\f[R]. @@ -30987,7 +31069,7 @@ reduce the sync run times for very large numbers of files. .PP The check may be run manually with \f[C]--check-sync=only\f[R]. It runs only the integrity check and terminates without actually -synching. +syncing. .PP Note that currently, \f[C]--check-sync\f[R] \f[B]only checks listing snapshots and NOT the actual files on the remotes.\f[R] Note also that @@ -31701,7 +31783,7 @@ flags are also supported. .SS How to filter directories .PP Filtering portions of the directory tree is a critical feature for -synching. +syncing. .PP Examples of directory trees (always beneath the Path1/Path2 root level) you may want to exclude from your sync: - Directory trees containing @@ -31859,7 +31941,7 @@ This noise can be quashed by adding \f[C]--quiet\f[R] to the bisync command line. .SS Example exclude-style filters files for use with Dropbox .IP \[bu] 2 -Dropbox disallows synching the listed temporary and configuration/data +Dropbox disallows syncing the listed temporary and configuration/data files. The \[ga]- \[ga] filters exclude these files where ever they may occur in the sync tree. @@ -32246,7 +32328,7 @@ single \f[C]-\f[R] or double dash. .SS Running tests .IP \[bu] 2 \f[C]go test . -case basic -remote local -remote2 local\f[R] runs the -\f[C]test_basic\f[R] test case using only the local filesystem, synching +\f[C]test_basic\f[R] test case using only the local filesystem, syncing one local directory with another local directory. Test script output is to the console, while commands within scenario.txt have their output sent to the \f[C].../workdir/test.log\f[R] file, which @@ -32579,6 +32661,10 @@ Also note a number of academic publications by Benjamin Pierce (http://www.cis.upenn.edu/%7Ebcpierce/papers/index.shtml#File%20Synchronization) about \f[I]Unison\f[R] and synchronization in general. .SS Changelog +.SS \f[C]v1.69.1\f[R] +.IP \[bu] 2 +Fixed an issue causing listings to not capture concurrent modifications +under certain conditions .SS \f[C]v1.68\f[R] .IP \[bu] 2 Fixed an issue affecting backends that round modtimes to a lower @@ -34293,7 +34379,7 @@ It assumes that \f[C]USER_NAME\f[R] has been created. The Resource entry must include both resource ARNs, as one implies the bucket and the other implies the bucket\[aq]s objects. .IP "3." 3 -When using s3-no-check-bucket and the bucket already exsits, the +When using s3-no-check-bucket and the bucket already exists, the \f[C]\[dq]arn:aws:s3:::BUCKET_NAME\[dq]\f[R] doesn\[aq]t have to be included. .PP @@ -38469,7 +38555,7 @@ location_constraint = au-nsw .PP Rclone can serve any remote over the S3 protocol. For details see the rclone serve -s3 (https://rclone.org/commands/rclone_serve_http/) documentation. +s3 (https://rclone.org/commands/rclone_serve_s3/) documentation. .PP For example, to serve \f[C]remote:path\f[R] over s3, run the server like this: @@ -38495,8 +38581,8 @@ use_multipart_uploads = false \f[R] .fi .PP -Note that setting \f[C]disable_multipart_uploads = true\f[R] is to work -around a bug (https://rclone.org/commands/rclone_serve_http/#bugs) which +Note that setting \f[C]use_multipart_uploads = false\f[R] is to work +around a bug (https://rclone.org/commands/rclone_serve_s3/#bugs) which will be fixed in due course. .SS Scaleway .PP @@ -39689,27 +39775,49 @@ Option endpoint. Endpoint for Linode Object Storage API. Choose a number from below, or type in your own value. Press Enter to leave empty. - 1 / Atlanta, GA (USA), us-southeast-1 + 1 / Amsterdam (Netherlands), nl-ams-1 + \[rs] (nl-ams-1.linodeobjects.com) + 2 / Atlanta, GA (USA), us-southeast-1 \[rs] (us-southeast-1.linodeobjects.com) - 2 / Chicago, IL (USA), us-ord-1 + 3 / Chennai (India), in-maa-1 + \[rs] (in-maa-1.linodeobjects.com) + 4 / Chicago, IL (USA), us-ord-1 \[rs] (us-ord-1.linodeobjects.com) - 3 / Frankfurt (Germany), eu-central-1 + 5 / Frankfurt (Germany), eu-central-1 \[rs] (eu-central-1.linodeobjects.com) - 4 / Milan (Italy), it-mil-1 + 6 / Jakarta (Indonesia), id-cgk-1 + \[rs] (id-cgk-1.linodeobjects.com) + 7 / London 2 (Great Britain), gb-lon-1 + \[rs] (gb-lon-1.linodeobjects.com) + 8 / Los Angeles, CA (USA), us-lax-1 + \[rs] (us-lax-1.linodeobjects.com) + 9 / Madrid (Spain), es-mad-1 + \[rs] (es-mad-1.linodeobjects.com) +10 / Melbourne (Australia), au-mel-1 + \[rs] (au-mel-1.linodeobjects.com) +11 / Miami, FL (USA), us-mia-1 + \[rs] (us-mia-1.linodeobjects.com) +12 / Milan (Italy), it-mil-1 \[rs] (it-mil-1.linodeobjects.com) - 5 / Newark, NJ (USA), us-east-1 +13 / Newark, NJ (USA), us-east-1 \[rs] (us-east-1.linodeobjects.com) - 6 / Paris (France), fr-par-1 +14 / Osaka (Japan), jp-osa-1 + \[rs] (jp-osa-1.linodeobjects.com) +15 / Paris (France), fr-par-1 \[rs] (fr-par-1.linodeobjects.com) - 7 / Seattle, WA (USA), us-sea-1 +16 / S\[~a]o Paulo (Brazil), br-gru-1 + \[rs] (br-gru-1.linodeobjects.com) +17 / Seattle, WA (USA), us-sea-1 \[rs] (us-sea-1.linodeobjects.com) - 8 / Singapore ap-south-1 +18 / Singapore, ap-south-1 \[rs] (ap-south-1.linodeobjects.com) - 9 / Stockholm (Sweden), se-sto-1 +19 / Singapore 2, sg-sin-1 + \[rs] (sg-sin-1.linodeobjects.com) +20 / Stockholm (Sweden), se-sto-1 \[rs] (se-sto-1.linodeobjects.com) -10 / Washington, DC, (USA), us-iad-1 +21 / Washington, DC, (USA), us-iad-1 \[rs] (us-iad-1.linodeobjects.com) -endpoint> 3 +endpoint> 5 Option acl. Canned ACL used when creating buckets and storing or copying objects. @@ -45295,7 +45403,7 @@ The nonce is incremented for each chunk read making sure each nonce is unique for each block written. The chance of a nonce being reused is minuscule. If you wrote an exabyte of data (10\[S1]\[u2078] bytes) you would have a -probability of approximately 2\[tmu]10\[u207B]\[S3]\[S2] of re-using a +probability of approximately 2\[tmu]10\[u207B]\[S3]\[S2] of reusing a nonce. .SS Chunk .PP @@ -54838,7 +54946,7 @@ Enter a value. config_2fa> 2FACODE Remote config -------------------- -[koofr] +[iclouddrive] - type: iclouddrive - apple_id: APPLEID - password: *** ENCRYPTED *** @@ -54854,6 +54962,28 @@ y/e/d> y .SS Advanced Data Protection .PP ADP is currently unsupported and need to be disabled +.PP +On iPhone, Settings \f[C]>\f[R] Apple Account \f[C]>\f[R] iCloud +\f[C]>\f[R] \[aq]Access iCloud Data on the Web\[aq] must be ON, and +\[aq]Advanced Data Protection\[aq] OFF. +.SS Troubleshooting +.SS Missing PCS cookies from the request +.PP +This means you have Advanced Data Protection (ADP) turned on. +This is not supported at the moment. +If you want to use rclone you will have to turn it off. +See above for how to turn it off. +.PP +You will need to clear the \f[C]cookies\f[R] and the +\f[C]trust_token\f[R] fields in the config. +Or you can delete the remote config and start again. +.PP +You should then run \f[C]rclone reconnect remote:\f[R]. +.PP +Note that changing the ADP setting may not take effect immediately - you +may need to wait a few hours or a day before you can get rclone to work +- keep clearing the config entry and running +\f[C]rclone reconnect remote:\f[R] until rclone functions properly. .SS Standard options .PP Here are the Standard options specific to iclouddrive (iCloud Drive). @@ -60946,7 +61076,7 @@ Microsoft Cloud for US Government \[dq]de\[dq] .RS 2 .IP \[bu] 2 -Microsoft Cloud Germany +Microsoft Cloud Germany (deprecated - try global region first). .RE .IP \[bu] 2 \[dq]cn\[dq] @@ -61951,6 +62081,43 @@ T} .TE .PP See the metadata (https://rclone.org/docs/#metadata) docs for more info. +.SS Impersonate other users as Admin +.PP +Unlike Google Drive and impersonating any domain user via service +accounts, OneDrive requires you to authenticate as an admin account, and +manually setup a remote per user you wish to impersonate. +.IP "1." 3 +In Microsoft 365 Admin Center (https://admin.microsoft.com), open each +user you need to \[dq]impersonate\[dq] and go to the OneDrive section. +There is a heading called \[dq]Get access to files\[dq], you need to +click to create the link, this creates the link of the format +\f[C]https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/\f[R] +but also changes the permissions so you your admin user has access. +.IP "2." 3 +Then in powershell run the following commands: +.IP +.nf +\f[C] +Install-Module Microsoft.Graph -Scope CurrentUser -Repository PSGallery -Force +Import-Module Microsoft.Graph.Files +Connect-MgGraph -Scopes \[dq]Files.ReadWrite.All\[dq] +# Follow the steps to allow access to your admin user +# Then run this for each user you want to impersonate to get the Drive ID +Get-MgUserDefaultDrive -UserId \[aq]{emailaddress}\[aq] +# This will give you output of the format: +# Name Id DriveType CreatedDateTime +# ---- -- --------- --------------- +# OneDrive b!XYZ123 business 14/10/2023 1:00:58\[u202F]pm +\f[R] +.fi +.IP "3." 3 +Then in rclone add a onedrive remote type, and use the +\f[C]Type in driveID\f[R] with the DriveID you got in the previous step. +One remote per user. +It will then confirm the drive ID, and hopefully give you a message of +\f[C]Found drive \[dq]root\[dq] of type \[dq]business\[dq]\f[R] and then +include the URL of the format +\f[C]https://{tenant}-my.sharepoint.com/personal/{user_name_domain_tld}/Documents\f[R] .SS Limitations .PP If you don\[aq]t use rclone for 90 days the refresh token will expire. @@ -74872,6 +75039,67 @@ Options: .IP \[bu] 2 \[dq]error\[dq]: return an error based on option value .SH Changelog +.SS v1.69.1 - 2025-02-14 +.PP +See commits (https://github.com/rclone/rclone/compare/v1.69.0...v1.69.1) +.IP \[bu] 2 +Bug Fixes +.RS 2 +.IP \[bu] 2 +lib/oauthutil: Fix redirect URL mismatch errors (Nick Craig-Wood) +.IP \[bu] 2 +bisync: Fix listings missing concurrent modifications (nielash) +.IP \[bu] 2 +serve s3: Fix list objects encoding-type (Nick Craig-Wood) +.IP \[bu] 2 +fs: Fix confusing \[dq]didn\[aq]t find section in config file\[dq] error +(Nick Craig-Wood) +.IP \[bu] 2 +doc fixes (Christoph Berger, Dimitri Papadopoulos, Matt Ickstadt, Nick +Craig-Wood, Tim White, Zachary Vorhies) +.IP \[bu] 2 +build: Added parallel docker builds and caching for go build in the +container (Anagh Kumar Baranwal) +.RE +.IP \[bu] 2 +VFS +.RS 2 +.IP \[bu] 2 +Fix the cache failing to upload symlinks when \f[C]--links\f[R] was +specified (Nick Craig-Wood) +.IP \[bu] 2 +Fix race detected by race detector (Nick Craig-Wood) +.IP \[bu] 2 +Close the change notify channel on Shutdown (izouxv) +.RE +.IP \[bu] 2 +B2 +.RS 2 +.IP \[bu] 2 +Fix \[dq]fatal error: concurrent map writes\[dq] (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Iclouddrive +.RS 2 +.IP \[bu] 2 +Add notes on ADP and Missing PCS cookies (Nick Craig-Wood) +.RE +.IP \[bu] 2 +Onedrive +.RS 2 +.IP \[bu] 2 +Mark German (de) region as deprecated (Nick Craig-Wood) +.RE +.IP \[bu] 2 +S3 +.RS 2 +.IP \[bu] 2 +Added new storage class to magalu provider (Bruno Fernandes) +.IP \[bu] 2 +Add DigitalOcean regions SFO2, LON1, TOR1, BLR1 (jkpe) +.IP \[bu] 2 +Add latest Linode Object Storage endpoints (jbagwell-akamai) +.RE .SS v1.69.0 - 2025-01-12 .PP See commits (https://github.com/rclone/rclone/compare/v1.68.0...v1.69.0) From 4f8dfd14fc9ad058d8aa9d43de9c2d6c43cd5592 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 14 Feb 2025 17:14:31 +0000 Subject: [PATCH 28/73] Start v1.69.2-DEV development --- VERSION | 2 +- docs/layouts/partials/version.html | 2 +- fs/versiontag.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index 5f8853011028d..48909b06dae93 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.69.1 +v1.69.2 diff --git a/docs/layouts/partials/version.html b/docs/layouts/partials/version.html index 003a53d3b0e53..7a35accbb921d 100644 --- a/docs/layouts/partials/version.html +++ b/docs/layouts/partials/version.html @@ -1 +1 @@ -v1.69.1 \ No newline at end of file +v1.69.2 \ No newline at end of file diff --git a/fs/versiontag.go b/fs/versiontag.go index 715d12ab0a0bd..be33950ca8cab 100644 --- a/fs/versiontag.go +++ b/fs/versiontag.go @@ -1,4 +1,4 @@ package fs // VersionTag of rclone -var VersionTag = "v1.69.1" +var VersionTag = "v1.69.2" From ff4d7c8dd04fef460c0b474bb0fad8c18565d62e Mon Sep 17 00:00:00 2001 From: hiddenmarten Date: Mon, 23 Dec 2024 16:41:24 +0100 Subject: [PATCH 29/73] rc: disable the metrics server when running `rclone rc` Fixes #8248 --- cmd/cmd.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 8f5f5f093e7c2..7e569c80be02f 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -429,11 +429,12 @@ func initConfig() { fs.Fatalf(nil, "Failed to start remote control: %v", err) } - // Start the metrics server if configured - _, err = rcserver.MetricsStart(ctx, &rc.Opt) - if err != nil { - fs.Fatalf(nil, "Failed to start metrics server: %v", err) - + // Start the metrics server if configured and not running the "rc" command + if os.Args[1] != "rc" { + _, err = rcserver.MetricsStart(ctx, &rc.Opt) + if err != nil { + fs.Fatalf(nil, "Failed to start metrics server: %v", err) + } } // Setup CPU profiling if desired From 6eb8919719ef52c548bb2ac1c4f467b288e972bc Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 11 Jan 2025 12:49:40 +0000 Subject: [PATCH 30/73] sync: fix cpu spinning when empty directory finding with leading slashes Before this change the logic which makes sure we create all directories could get confused with directories which started with slashes and get into an infinite loop consuming 100% of the CPU. --- fs/sync/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/sync/sync.go b/fs/sync/sync.go index e0a3e23ac41bb..7feedfcd770e9 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -726,7 +726,7 @@ func (s *syncCopyMove) markParentNotEmpty(entry fs.DirEntry) { parentDir = "" } delete(s.srcEmptyDirs, parentDir) - if parentDir == "" { + if parentDir == "" || parentDir == "/" { break } parentDir = path.Dir(parentDir) From c62fbc5269788db5cf91f523557dfa63eb630de3 Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Tue, 17 Sep 2024 18:09:54 +0530 Subject: [PATCH 31/73] stats: fix the speed not getting updated after a pause in the processing This shifts the behavior of the average loop to be a persistent loop that gets resumed/paused when transfers & checks are started/completed. Previously, the averageLoop was stopped on completion of transfers & checks but failed to start again due to the protection of the sync.Once Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- fs/accounting/stats.go | 113 +++++++++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 38 deletions(-) diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go index 221ab30012150..8a4526fab179e 100644 --- a/fs/accounting/stats.go +++ b/fs/accounting/stats.go @@ -65,28 +65,29 @@ type StatsInfo struct { } type averageValues struct { - mu sync.Mutex - lpBytes int64 - lpTime time.Time - speed float64 - stop chan bool - stopped sync.WaitGroup - startOnce sync.Once - stopOnce sync.Once + mu sync.Mutex + lpBytes int64 + lpTime time.Time + speed float64 + stop chan bool + stopped sync.WaitGroup + started bool } // NewStats creates an initialised StatsInfo func NewStats(ctx context.Context) *StatsInfo { ci := fs.GetConfig(ctx) - return &StatsInfo{ + s := &StatsInfo{ ctx: ctx, ci: ci, checking: newTransferMap(ci.Checkers, "checking"), transferring: newTransferMap(ci.Transfers, "transferring"), inProgress: newInProgress(ctx), startTime: time.Now(), - average: averageValues{stop: make(chan bool)}, + average: averageValues{}, } + s.startAverageLoop() + return s } // RemoteStats returns stats for rc @@ -328,61 +329,96 @@ func (s *StatsInfo) averageLoop() { ticker := time.NewTicker(averagePeriodLength) defer ticker.Stop() - startTime := time.Now() a := &s.average defer a.stopped.Done() + + shouldRun := false + for { select { case now := <-ticker.C: a.mu.Lock() - var elapsed float64 - if a.lpTime.IsZero() { - elapsed = now.Sub(startTime).Seconds() - } else { - elapsed = now.Sub(a.lpTime).Seconds() + + if !shouldRun { + a.mu.Unlock() + continue } + avg := 0.0 + elapsed := now.Sub(a.lpTime).Seconds() if elapsed > 0 { avg = float64(a.lpBytes) / elapsed } + if period < averagePeriod { period++ } + a.speed = (avg + a.speed*(period-1)) / period a.lpBytes = 0 a.lpTime = now + + a.mu.Unlock() + + case stop, ok := <-a.stop: + if !ok { + return // Channel closed, exit the loop + } + + a.mu.Lock() + + // If we are resuming, store the current time + if !shouldRun && !stop { + a.lpTime = time.Now() + } + shouldRun = !stop + a.mu.Unlock() - case <-a.stop: - return } } } -// Start the average loop -func (s *StatsInfo) startAverageLoop() { +// Resume the average loop +func (s *StatsInfo) resumeAverageLoop() { + s.mu.Lock() + defer s.mu.Unlock() + s.average.stop <- false +} + +// Pause the average loop +func (s *StatsInfo) pauseAverageLoop() { s.mu.Lock() defer s.mu.Unlock() - s.average.startOnce.Do(func() { + s.average.stop <- true +} + +// Start the average loop +// +// Call with the mutex held +func (s *StatsInfo) _startAverageLoop() { + if !s.average.started { + s.average.stop = make(chan bool) + s.average.started = true s.average.stopped.Add(1) go s.averageLoop() - }) + } +} + +// Start the average loop +func (s *StatsInfo) startAverageLoop() { + s.mu.Lock() + defer s.mu.Unlock() + s._startAverageLoop() } // Stop the average loop // // Call with the mutex held func (s *StatsInfo) _stopAverageLoop() { - s.average.stopOnce.Do(func() { + if s.average.started { close(s.average.stop) s.average.stopped.Wait() - }) -} - -// Stop the average loop -func (s *StatsInfo) stopAverageLoop() { - s.mu.Lock() - defer s.mu.Unlock() - s._stopAverageLoop() + } } // String convert the StatsInfo to a string for printing @@ -564,9 +600,9 @@ func (s *StatsInfo) GetBytesWithPending() int64 { pending := int64(0) for _, tr := range s.startedTransfers { if tr.acc != nil { - bytes, size := tr.acc.progress() - if bytes < size { - pending += size - bytes + bytesRead, size := tr.acc.progress() + if bytesRead < size { + pending += size - bytesRead } } } @@ -699,7 +735,8 @@ func (s *StatsInfo) ResetCounters() { s.oldDuration = 0 s._stopAverageLoop() - s.average = averageValues{stop: make(chan bool)} + s.average = averageValues{} + s._startAverageLoop() } // ResetErrors sets the errors count to 0 and resets lastError, fatalError and retryError @@ -788,7 +825,7 @@ func (s *StatsInfo) NewTransfer(obj fs.DirEntry, dstFs fs.Fs) *Transfer { } tr := newTransfer(s, obj, srcFs, dstFs) s.transferring.add(tr) - s.startAverageLoop() + s.resumeAverageLoop() return tr } @@ -796,7 +833,7 @@ func (s *StatsInfo) NewTransfer(obj fs.DirEntry, dstFs fs.Fs) *Transfer { func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64, srcFs, dstFs fs.Fs) *Transfer { tr := newTransferRemoteSize(s, remote, size, false, "", srcFs, dstFs) s.transferring.add(tr) - s.startAverageLoop() + s.resumeAverageLoop() return tr } @@ -811,7 +848,7 @@ func (s *StatsInfo) DoneTransferring(remote string, ok bool) { s.mu.Unlock() } if s.transferring.empty() && s.checking.empty() { - time.AfterFunc(averageStopAfter, s.stopAverageLoop) + s.pauseAverageLoop() } } From 57bbb4be9f6c2eb8753b5b274e82f7f6e10a9875 Mon Sep 17 00:00:00 2001 From: ll3006 Date: Fri, 7 Feb 2025 14:23:28 +0100 Subject: [PATCH 32/73] sync: copy dir modtimes even when copyEmptySrcDirs is false - fixes #8317 Before, after a sync, only file modtimes were updated when not using --copy-empty-src-dirs. This ensures modtimes are updated to match the source folder, regardless of copyEmptySrcDir. The flag --no-update-dir-modtime (which previously did nothing) will disable this. --- fs/sync/sync.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 7feedfcd770e9..e30c86ed2ff00 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -1109,23 +1109,17 @@ func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Dire if !s.setDirModTimeAfter && equal { return nil } - if s.setDirModTimeAfter && equal { - newDst = dst - } else if s.copyEmptySrcDirs { - if s.setDirMetadata { + newDst = dst + if !equal { + if s.setDirMetadata && s.copyEmptySrcDirs { newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src) - } else if s.setDirModTime { - if dst == nil { - newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx)) - } else { - newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx)) - } - } else if dst == nil { - // Create the directory if it doesn't exist + } else if dst == nil && s.setDirModTime && s.copyEmptySrcDirs { + newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx)) + } else if dst == nil && s.copyEmptySrcDirs { err = operations.Mkdir(ctx, f, dir) + } else if dst != nil && s.setDirModTime { + newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx)) } - } else { - newDst = dst } // If we need to set modtime after and we created a dir, then save it for later if s.setDirModTime && s.setDirModTimeAfter && err == nil { From 92efc5ff43900fc8505027d8de3974da8ac1cdb1 Mon Sep 17 00:00:00 2001 From: Janne Hellsten Date: Wed, 12 Feb 2025 23:31:05 +0200 Subject: [PATCH 33/73] cmd: fix crash if rclone is invoked without any arguments - Fixes #8378 --- cmd/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 7e569c80be02f..545da371d5ec3 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -430,7 +430,7 @@ func initConfig() { } // Start the metrics server if configured and not running the "rc" command - if os.Args[1] != "rc" { + if len(os.Args) >= 2 && os.Args[1] != "rc" { _, err = rcserver.MetricsStart(ctx, &rc.Opt) if err != nil { fs.Fatalf(nil, "Failed to start metrics server: %v", err) From b00dcc37bd477fa1f6927792ed378c56f514f3ab Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Thu, 13 Feb 2025 17:09:24 +0530 Subject: [PATCH 34/73] accounting: fix percentDiff calculation -- fixes #8345 Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- fs/accounting/stats_groups_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fs/accounting/stats_groups_test.go b/fs/accounting/stats_groups_test.go index 8c02229e736f7..b7a817824f4a2 100644 --- a/fs/accounting/stats_groups_test.go +++ b/fs/accounting/stats_groups_test.go @@ -237,5 +237,14 @@ func TestCountError(t *testing.T) { } func percentDiff(start, end uint64) uint64 { - return (start - end) * 100 / start + if start == 0 { + return 0 // Handle zero start value to avoid division by zero + } + var diff uint64 + if end > start { + diff = end - start // Handle case where end is larger than start + } else { + diff = start - end + } + return (diff * 100) / start } From 5fa9c0209eaa7c6308938d78530862620e6042da Mon Sep 17 00:00:00 2001 From: Michael Kebe Date: Sat, 15 Feb 2025 11:11:26 +0100 Subject: [PATCH 35/73] docs: improved sftp limitations Added a link to `--sftp-path-override` for a better solution with working hash calculation. --- docs/content/sftp.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/content/sftp.md b/docs/content/sftp.md index 56f4d98dc6a14..5da567247da4d 100644 --- a/docs/content/sftp.md +++ b/docs/content/sftp.md @@ -1108,7 +1108,8 @@ Properties: On some SFTP servers (e.g. Synology) the paths are different for SSH and SFTP so the hashes can't be calculated properly. -For them using `disable_hashcheck` is a good idea. +You can either use [`--sftp-path-override`](#--sftp-path-override) +or [`disable_hashcheck`](#--sftp-disable-hashcheck). The only ssh agent supported under Windows is Putty's pageant. From 861456d9703a41cc7f95f63abe8725cff585c588 Mon Sep 17 00:00:00 2001 From: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> Date: Sat, 15 Feb 2025 19:53:43 +0530 Subject: [PATCH 36/73] build: fix docker plugin build - fixes #8394 Signed-off-by: Anagh Kumar Baranwal <6824881+darthShadow@users.noreply.github.com> --- .../workflows/build_publish_docker_plugin.yml | 72 ++++++++++--------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build_publish_docker_plugin.yml b/.github/workflows/build_publish_docker_plugin.yml index 4056cb04be316..42af5265c7d14 100644 --- a/.github/workflows/build_publish_docker_plugin.yml +++ b/.github/workflows/build_publish_docker_plugin.yml @@ -7,39 +7,43 @@ name: Release Build for Docker Plugin on: release: types: [published] + workflow_dispatch: + inputs: + manual: + description: Manual run (bypass default conditions) + type: boolean + default: true jobs: - - build_docker_volume_plugin: - if: github.repository == 'rclone/rclone' - needs: build - runs-on: ubuntu-latest - name: Build docker plugin job - steps: - - name: Free some space - shell: bash - run: | - df -h . - # Remove android SDK - sudo rm -rf /usr/local/lib/android || true - # Remove .net runtime - sudo rm -rf /usr/share/dotnet || true - df -h . - - name: Checkout master - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Build and publish docker plugin - shell: bash - run: | - VER=${GITHUB_REF#refs/tags/} - PLUGIN_USER=rclone - docker login --username ${{ secrets.DOCKER_HUB_USER }} \ - --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" - for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do - export PLUGIN_USER PLUGIN_ARCH - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} - make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} - done - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest - make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} + build_docker_volume_plugin: + if: inputs.manual || github.repository == 'rclone/rclone' + name: Build docker plugin job + runs-on: ubuntu-latest + steps: + - name: Free some space + shell: bash + run: | + df -h . + # Remove android SDK + sudo rm -rf /usr/local/lib/android || true + # Remove .net runtime + sudo rm -rf /usr/share/dotnet || true + df -h . + - name: Checkout master + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build and publish docker plugin + shell: bash + run: | + VER=${GITHUB_REF#refs/tags/} + PLUGIN_USER=rclone + docker login --username ${{ secrets.DOCKER_HUB_USER }} \ + --password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}" + for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do + export PLUGIN_USER PLUGIN_ARCH + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-} + make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v} + done + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=latest + make docker-plugin PLUGIN_ARCH=amd64 PLUGIN_TAG=${VER#v} From 286ae6a1b4deeea8677614e15c37dde4e1796ff8 Mon Sep 17 00:00:00 2001 From: Lorenz Brun Date: Tue, 20 Sep 2022 00:31:56 +0200 Subject: [PATCH 37/73] vfs: fix directory cache serving stale data The VFS directory cache layer didn't update directory entry properties if they are reused after cache invalidation. Update them unconditionally as newDir sets them to the same value and setting a pointer is cheaper in both LoC as well as CPU cycles than a branch. Also add a test exercising this behavior. Fixes #6335 --- vfs/dir.go | 1 + vfs/dir_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/vfs/dir.go b/vfs/dir.go index c72494eee80b3..a4a31c4b6891c 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -760,6 +760,7 @@ func (d *Dir) _readDirFromEntries(entries fs.DirEntries, dirTree dirtree.DirTree dir := node.(*Dir) dir.mu.Lock() dir.modTime = item.ModTime(context.TODO()) + dir.entry = item if dirTree != nil { err = dir._readDirFromDirTree(dirTree, when) if err != nil { diff --git a/vfs/dir_test.go b/vfs/dir_test.go index 555cacf1ffc49..69e45c269cedf 100644 --- a/vfs/dir_test.go +++ b/vfs/dir_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "runtime" "sort" "testing" "time" @@ -655,3 +656,34 @@ func TestDirFileOpen(t *testing.T) { require.NoError(t, err) assert.Equal(t, int64(12), fi.Size()) } + +func TestDirEntryModTimeInvalidation(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("dirent modtime is unreliable on Windows filesystems") + } + r, vfs := newTestVFS(t) + + // Needs to be less than 2x the wait time below, othewrwise the entry + // gets cleared out before it had a chance to be updated. + vfs.Opt.DirCacheTime = fs.Duration(50 * time.Millisecond) + + r.WriteObject(context.Background(), "dir/file1", "file1 contents", t1) + + node, err := vfs.Stat("dir") + require.NoError(t, err) + modTime1 := node.(*Dir).DirEntry().ModTime(context.Background()) + + // Wait some time, then write another file which must update ModTime of + // the directory. + time.Sleep(75 * time.Millisecond) + r.WriteObject(context.Background(), "dir/file2", "file2 contents", t2) + + node2, err := vfs.Stat("dir") + require.NoError(t, err) + modTime2 := node2.(*Dir).DirEntry().ModTime(context.Background()) + + // ModTime of directory must be different after second file was written. + if modTime1.Equal(modTime2) { + t.Error("ModTime not invalidated") + } +} From 1ac7e323029c9822bfe4e9ad0635d9baabd454f6 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 15 Feb 2025 20:08:33 +0000 Subject: [PATCH 38/73] docs: correct stable release workflow --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index c4c7d239e8b86..e455eb23e9924 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -124,8 +124,8 @@ Now * git co ${BASE_TAG}-stable * git cherry-pick any fixes - * Do the steps as above * make startstable + * Do the steps as above * git co master * `#` cherry pick the changes to the changelog - check the diff to make sure it is correct * git checkout ${BASE_TAG}-stable docs/content/changelog.md From 659aca871141e4ea7c93547346ed450c589fb630 Mon Sep 17 00:00:00 2001 From: emyarod Date: Sat, 15 Feb 2025 18:07:20 -0600 Subject: [PATCH 39/73] chore: update contributor email --- bin/.ignore-emails | 3 ++- docs/content/authors.md | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/.ignore-emails b/bin/.ignore-emails index 9b364fae1acb8..fb53d92f5a742 100644 --- a/bin/.ignore-emails +++ b/bin/.ignore-emails @@ -11,4 +11,5 @@ - \ No newline at end of file + + \ No newline at end of file diff --git a/docs/content/authors.md b/docs/content/authors.md index ed83b05cd4634..3eab5328ee7bf 100644 --- a/docs/content/authors.md +++ b/docs/content/authors.md @@ -809,7 +809,6 @@ put them back in again.` >}} * ben-ba * Eli Orzitzer * Anthony Metzidis - * emyarod * keongalvin * rarspace01 * Paul Stern From 6a9c2350de301efa8ef70bfbfc53bdbe7b7b51a5 Mon Sep 17 00:00:00 2001 From: Dave Vasilevsky Date: Sat, 15 Feb 2025 19:26:09 -0500 Subject: [PATCH 40/73] dropbox: Retry link without expiry Dropbox only allows public links with expiry for certain account types. Rather than erroring for other accounts, retry without expiry. --- backend/dropbox/dropbox.go | 10 ++++++++++ fstest/test_all/config.yaml | 4 ---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 82eee8e692713..fb978ec0cb7bd 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -1174,6 +1174,16 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, return shouldRetry(ctx, err) }) + if err != nil && createArg.Settings.Expires != nil && strings.Contains(err.Error(), sharing.SharedLinkSettingsErrorNotAuthorized) { + // Some plans can't create links with expiry + fs.Debugf(absPath, "can't create link with expiry, trying without") + createArg.Settings.Expires = nil + err = f.pacer.Call(func() (bool, error) { + linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg) + return shouldRetry(ctx, err) + }) + } + if err != nil && strings.Contains(err.Error(), sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) { fs.Debugf(absPath, "has a public link already, attempting to retrieve it") diff --git a/fstest/test_all/config.yaml b/fstest/test_all/config.yaml index 624d875d5c5ce..c097ca6a65824 100644 --- a/fstest/test_all/config.yaml +++ b/fstest/test_all/config.yaml @@ -144,10 +144,6 @@ backends: - backend: "dropbox" remote: "TestDropbox:" fastlist: false - ignore: - # This test doesn't work on a standard dropbox account because it - # tries to set the expiry of the link - - TestIntegration/FsMkdir/FsPutFiles/PublicLink # - backend: "filefabric" # remote: "TestFileFabric:" # fastlist: false From d725f4f4def1b329f4071d2b873ca200f8a6c5bd Mon Sep 17 00:00:00 2001 From: Zachary Vorhies Date: Tue, 4 Mar 2025 09:31:56 -0700 Subject: [PATCH 41/73] docs: add note for using rclone cat for slicing out a byte range from a file --- cmd/copyto/copyto.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/copyto/copyto.go b/cmd/copyto/copyto.go index 90e8584816847..edc793071e4f5 100644 --- a/cmd/copyto/copyto.go +++ b/cmd/copyto/copyto.go @@ -43,6 +43,8 @@ This doesn't transfer files that are identical on src and dst, testing by size and modification time or MD5SUM. It doesn't delete files from the destination. +*If you are looking to copy just a byte range of a file, please see 'rclone cat --offset X --count Y'* + **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics `, Annotations: map[string]string{ From 2e853b6e75c8c799340358bcf558778d442ee6cd Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 5 Mar 2025 11:08:26 +0000 Subject: [PATCH 42/73] serve nfs: fix unlikely crash --- cmd/serve/nfs/filesystem.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/serve/nfs/filesystem.go b/cmd/serve/nfs/filesystem.go index fe30326cb3846..a87563171b929 100644 --- a/cmd/serve/nfs/filesystem.go +++ b/cmd/serve/nfs/filesystem.go @@ -29,6 +29,7 @@ func setSys(fi os.FileInfo) { node, ok := fi.(vfs.Node) if !ok { fs.Errorf(fi, "internal error: %T is not a vfs.Node", fi) + return } vfs := node.VFS() // Set the UID and GID for the node passed in from the VFS defaults. From ef99b9ad18a7ac6c9775df4d0d4cf6ca67a7a17b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Fri, 28 Feb 2025 17:01:00 +0000 Subject: [PATCH 43/73] fs/object: fix memory object out of bounds Seek --- fs/object/object.go | 13 ++++++++++--- fs/object/object_test.go | 20 +++++++++++++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/fs/object/object.go b/fs/object/object.go index 40aee8dec279b..9b4e25afe5018 100644 --- a/fs/object/object.go +++ b/fs/object/object.go @@ -279,19 +279,26 @@ func (o *MemoryObject) SetModTime(ctx context.Context, modTime time.Time) error // Open opens the file for read. Call Close() on the returned io.ReadCloser func (o *MemoryObject) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) { - content := o.content + var offset, limit int64 = 0, -1 for _, option := range options { switch x := option.(type) { case *fs.RangeOption: - content = o.content[x.Start:x.End] + offset, limit = x.Decode(o.Size()) case *fs.SeekOption: - content = o.content[x.Offset:] + offset = x.Offset default: if option.Mandatory() { fs.Logf(o, "Unsupported mandatory option: %v", option) } } } + content := o.content + offset = max(offset, 0) + if limit < 0 { + content = content[offset:] + } else { + content = content[offset:min(offset+limit, int64(len(content)))] + } return io.NopCloser(bytes.NewBuffer(content)), nil } diff --git a/fs/object/object_test.go b/fs/object/object_test.go index 2a423627281b6..be90c771e4c56 100644 --- a/fs/object/object_test.go +++ b/fs/object/object_test.go @@ -110,6 +110,7 @@ func TestMemoryObject(t *testing.T) { assert.Equal(t, newNow, o.ModTime(context.Background())) checkOpen := func(rc io.ReadCloser, expected string) { + t.Helper() actual, err := io.ReadAll(rc) assert.NoError(t, err) err = rc.Close() @@ -118,6 +119,7 @@ func TestMemoryObject(t *testing.T) { } checkContent := func(o fs.Object, expected string) { + t.Helper() rc, err := o.Open(context.Background()) assert.NoError(t, err) checkOpen(rc, expected) @@ -127,12 +129,28 @@ func TestMemoryObject(t *testing.T) { rc, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 3}) assert.NoError(t, err) - checkOpen(rc, "ot") + checkOpen(rc, "ota") + + rc, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: -1}) + assert.NoError(t, err) + checkOpen(rc, "otato") + + rc, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 4096}) + assert.NoError(t, err) + checkOpen(rc, "otato") + + rc, err = o.Open(context.Background(), &fs.RangeOption{Start: -1, End: 4}) + assert.NoError(t, err) + checkOpen(rc, "tato") rc, err = o.Open(context.Background(), &fs.SeekOption{Offset: 3}) assert.NoError(t, err) checkOpen(rc, "ato") + rc, err = o.Open(context.Background(), &fs.SeekOption{Offset: -100}) + assert.NoError(t, err) + checkOpen(rc, "potato") + // check it fits within the buffer newNow = now.Add(2 * time.Minute) newContent := bytes.NewBufferString("Rutabaga") From 296c26b671e7b3d2c3433e82a1af4eb085664aad Mon Sep 17 00:00:00 2001 From: jack <9480542+jackusm@users.noreply.github.com> Date: Sat, 8 Mar 2025 22:40:46 +0000 Subject: [PATCH 44/73] docs: fix incorrect mentions of vfs-cache-min-free-size --- vfs/vfs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vfs/vfs.md b/vfs/vfs.md index 3d0d5622cd877..dde8f9f30ee8a 100644 --- a/vfs/vfs.md +++ b/vfs/vfs.md @@ -95,11 +95,11 @@ seconds. If rclone is quit or dies with files that haven't been uploaded, these will be uploaded next time rclone is run with the same flags. -If using `--vfs-cache-max-size` or `--vfs-cache-min-free-size` note +If using `--vfs-cache-max-size` or `--vfs-cache-min-free-space` note that the cache may exceed these quotas for two reasons. Firstly because it is only checked every `--vfs-cache-poll-interval`. Secondly because open files cannot be evicted from the cache. When -`--vfs-cache-max-size` or `--vfs-cache-min-free-size` is exceeded, +`--vfs-cache-max-size` or `--vfs-cache-min-free-space` is exceeded, rclone will attempt to evict the least accessed files from the cache first. rclone will start with files that haven't been accessed for the longest. This cache flushing strategy is efficient and more relevant From e7c46912a3176e0350f3fbf17c0d91a8ea2681e9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 12 Mar 2025 17:29:53 +0000 Subject: [PATCH 45/73] fs: fix corruption of SizeSuffix with "B" suffix in config (eg --min-size) Before this change, the config system round tripped fs.SizeSuffix values through strings like this, corrupting them in the process. "2B" -> 2 -> "2" -> 2048 This caused `--min-size 2B` to be interpreted as `--min-size 2k`. This fix makes sure SizeSuffix values have a "B" suffix when turned into a string where necessary, so it becomes "2B" -> 2 -> "2B" -> 2 In rclone v2 we should probably declare unsuffixed SizeSuffix values are in bytes not kBytes (done for rsync compatibility) but this would be a backwards incompatible change which we don't want for v1. Fixes #8437 Fixes #8212 Fixes #5169 --- fs/fs_test.go | 28 ++++++++++++++++++++++++++++ fs/registry.go | 18 +++++++++++++++--- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/fs/fs_test.go b/fs/fs_test.go index e0099e96a98cf..45b16229c907e 100644 --- a/fs/fs_test.go +++ b/fs/fs_test.go @@ -261,6 +261,34 @@ func TestOptionString(t *testing.T) { assert.Equal(t, "true", caseInsensitiveOption.String()) } +func TestOptionStringStringArray(t *testing.T) { + opt := Option{ + Name: "string_array", + Default: []string(nil), + } + assert.Equal(t, "", opt.String()) + opt.Default = []string{} + assert.Equal(t, "", opt.String()) + opt.Default = []string{"a", "b"} + assert.Equal(t, "a,b", opt.String()) + opt.Default = []string{"hello, world!", "goodbye, world!"} + assert.Equal(t, `"hello, world!","goodbye, world!"`, opt.String()) +} + +func TestOptionStringSizeSuffix(t *testing.T) { + opt := Option{ + Name: "size_suffix", + Default: SizeSuffix(0), + } + assert.Equal(t, "0", opt.String()) + opt.Default = SizeSuffix(-1) + assert.Equal(t, "off", opt.String()) + opt.Default = SizeSuffix(100) + assert.Equal(t, "100B", opt.String()) + opt.Default = SizeSuffix(1024) + assert.Equal(t, "1Ki", opt.String()) +} + func TestOptionSet(t *testing.T) { o := caseInsensitiveOption assert.Equal(t, true, o.Value) diff --git a/fs/registry.go b/fs/registry.go index e341bd2d57dc8..618112db3959d 100644 --- a/fs/registry.go +++ b/fs/registry.go @@ -258,15 +258,27 @@ func (o *Option) IsDefault() bool { // String turns Option into a string func (o *Option) String() string { v := o.GetValue() - if stringArray, isStringArray := v.([]string); isStringArray { + switch x := v.(type) { + case []string: // Treat empty string array as empty string // This is to make the default value of the option help nice - if len(stringArray) == 0 { + if len(x) == 0 { return "" } // Encode string arrays as CSV // The default Go encoding can't be decoded uniquely - return CommaSepList(stringArray).String() + return CommaSepList(x).String() + case SizeSuffix: + str := x.String() + // Suffix bare numbers with "B" unless they are 0 + // + // This makes sure that fs.SizeSuffix roundtrips through string + if len(str) > 0 && str != "0" { + if lastDigit := str[len(str)-1]; lastDigit >= '0' && lastDigit <= '9' { + str += "B" + } + } + return str } return fmt.Sprint(v) } From f46787dcc353d37b1a4187531066a330e0adbbab Mon Sep 17 00:00:00 2001 From: nielash Date: Thu, 6 Mar 2025 08:15:51 -0500 Subject: [PATCH 46/73] http: correct root if definitely pointing to a file - fixes #8428 This was formalized in https://github.com/rclone/rclone/commit/c69eb84573c85206ab028eda2987180e049ef2e4 But it appears that we forgot to update `http`, and the `FsRoot` test didn't catch it because we don't currently have an http integration test. --- backend/http/http.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/http/http.go b/backend/http/http.go index 60c6cb99c082e..9f1aa6994778e 100644 --- a/backend/http/http.go +++ b/backend/http/http.go @@ -180,7 +180,6 @@ func getFsEndpoint(ctx context.Context, client *http.Client, url string, opt *Op } addHeaders(req, opt) res, err := noRedir.Do(req) - if err != nil { fs.Debugf(nil, "Assuming path is a file as HEAD request could not be sent: %v", err) return createFileResult() @@ -249,6 +248,14 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err f.httpClient = client f.endpoint = u f.endpointURL = u.String() + + if isFile { + // Correct root if definitely pointing to a file + f.root = path.Dir(f.root) + if f.root == "." || f.root == "/" { + f.root = "" + } + } return isFile, nil } From 46dcc0eaf6a9d92e80b18b4da36fda8caddc2f99 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 25 Mar 2025 15:30:49 +0000 Subject: [PATCH 47/73] rc: fix debug/* commands not being available over unix sockets This was caused by an incorrect handler URL which was passing the debug/* commands to the debug/pprof handler by accident. This only happened when using unix sockets. --- fs/rc/rcserver/rcserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go index 8cc666f3c2142..b38d5ec9efcc3 100644 --- a/fs/rc/rcserver/rcserver.go +++ b/fs/rc/rcserver/rcserver.go @@ -123,7 +123,7 @@ func newServer(ctx context.Context, opt *rc.Options, mux *http.ServeMux) (*Serve ) // Add the debug handler which is installed in the default mux - router.Handle("/debug/*", mux) + router.Handle("/debug/pprof/*", mux) // FIXME split these up into individual functions router.Get("/*", s.handler) From f76c72b5cc5085d5ed403a7521e3990090419c73 Mon Sep 17 00:00:00 2001 From: alingse Date: Fri, 4 Apr 2025 22:21:45 +0800 Subject: [PATCH 48/73] operations: fix call fmt.Errorf with wrong err --- fs/operations/multithread.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/operations/multithread.go b/fs/operations/multithread.go index 582df84459ca6..c11e50d17c4bc 100644 --- a/fs/operations/multithread.go +++ b/fs/operations/multithread.go @@ -315,8 +315,8 @@ func (w *writerAtChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, r // if we were buffering, flush to disk switch w := writer.(type) { case *bufio.Writer: - er2 := w.Flush() - if er2 != nil { + err = w.Flush() + if err != nil { return -1, fmt.Errorf("multi-thread copy: flush failed: %w", err) } } From f52d7533634f686c00edb9703cacd9fdbabb954e Mon Sep 17 00:00:00 2001 From: eccoisle <167755281+eccoisle@users.noreply.github.com> Date: Sun, 6 Apr 2025 22:28:34 +1000 Subject: [PATCH 49/73] docs: replace option --auto-filename-header with --header-filename --- cmd/copyurl/copyurl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/copyurl/copyurl.go b/cmd/copyurl/copyurl.go index c5f46cbc70078..6c5ec34c1bb1e 100644 --- a/cmd/copyurl/copyurl.go +++ b/cmd/copyurl/copyurl.go @@ -43,7 +43,7 @@ Setting |--auto-filename| will attempt to automatically determine the filename from the URL (after any redirections) and used in the destination path. -With |--auto-filename-header| in addition, if a specific filename is +With |--header-filename| in addition, if a specific filename is set in HTTP headers, it will be used instead of the name from the URL. With |--print-filename| in addition, the resulting file name will be printed. From 85c156370152e40d3a3acd9fc0ea28adae9bdfee Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 5 Apr 2025 13:04:13 +0100 Subject: [PATCH 50/73] docs: Add rcloneui.com as Silver Sponsor --- docs/layouts/chrome/menu.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/layouts/chrome/menu.html b/docs/layouts/chrome/menu.html index 624ce12d9187c..3baafa102ff73 100644 --- a/docs/layouts/chrome/menu.html +++ b/docs/layouts/chrome/menu.html @@ -28,6 +28,14 @@
      +
      +
      + Silver Sponsor +
      +
      +
      +
      +
      {{end}}
      From b12d2f32db27f877bbdce417126f6d47b4ec4a98 Mon Sep 17 00:00:00 2001 From: Danny Garside Date: Tue, 8 Apr 2025 20:51:22 +0100 Subject: [PATCH 51/73] docs: fix minor typo in box docs --- docs/content/box.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/box.md b/docs/content/box.md index ff197024eca75..651f407111b83 100644 --- a/docs/content/box.md +++ b/docs/content/box.md @@ -87,7 +87,7 @@ machine with no Internet browser available. Note that rclone runs a webserver on your local machine to collect the token as returned from Box. This only runs from the moment it opens your browser to the moment you get back the verification code. This -is on `http://127.0.0.1:53682/` and this it may require you to unblock +is on `http://127.0.0.1:53682/` and this may require you to unblock it temporarily if you are running a host firewall. Once configured you can then use `rclone` like this, From a3c4e2fd206972ad42c2b5a84474a31f22f78f2b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 5 Apr 2025 13:35:04 +0100 Subject: [PATCH 52/73] lib/http: fix race between Serve() and Shutdown() This was discovered by the race detector. --- lib/http/server.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/http/server.go b/lib/http/server.go index ce13b82b66557..0993759ee6331 100644 --- a/lib/http/server.go +++ b/lib/http/server.go @@ -229,7 +229,8 @@ type Server struct { cfg Config template *TemplateConfig htmlTemplate *template.Template - usingAuth bool // set if we are using auth middleware + usingAuth bool // set if we are using auth middleware + mu sync.Mutex // mutex protects RW variables below atexitHandle atexit.FnHandle } @@ -517,7 +518,9 @@ func (s *Server) Serve() { go ii.serve(&s.wg) } // Install an atexit handler to shutdown gracefully + s.mu.Lock() s.atexitHandle = atexit.Register(func() { _ = s.Shutdown() }) + s.mu.Unlock() } // Wait blocks while the server is serving requests @@ -536,10 +539,12 @@ const gracefulShutdownTime = 10 * time.Second // Shutdown gracefully shuts down the server func (s *Server) Shutdown() error { // Stop the atexit handler + s.mu.Lock() if s.atexitHandle != nil { atexit.Unregister(s.atexitHandle) s.atexitHandle = nil } + s.mu.Unlock() for _, ii := range s.instances { expiry := time.Now().Add(gracefulShutdownTime) ctx, cancel := context.WithDeadline(context.Background(), expiry) From ab5844df77cfde80012dddacff1de54a60c4fe97 Mon Sep 17 00:00:00 2001 From: Samantha Bowen Date: Sat, 29 Mar 2025 14:30:17 -0700 Subject: [PATCH 53/73] fs/config: Read configuration passwords from stdin even when terminated with EOF - fixes #8480 --- fs/config/ui.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/config/ui.go b/fs/config/ui.go index fcee469b13ff8..8edf5350b7657 100644 --- a/fs/config/ui.go +++ b/fs/config/ui.go @@ -7,6 +7,7 @@ import ( "context" "errors" "fmt" + "io" "os" "sort" "strconv" @@ -27,7 +28,7 @@ import ( var ReadLine = func() string { buf := bufio.NewReader(os.Stdin) line, err := buf.ReadString('\n') - if err != nil { + if err != nil && (line == "" || err != io.EOF) { fs.Fatalf(nil, "Failed to read line: %v", err) } return strings.TrimSpace(line) From d8c7031dc36bd5857364c3c98ce9dc3c413d8b90 Mon Sep 17 00:00:00 2001 From: simonmcnair <101189766+simonmcnair@users.noreply.github.com> Date: Wed, 9 Apr 2025 11:54:06 +0100 Subject: [PATCH 54/73] docs: update fuse version in docker docs --- docs/content/docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/docker.md b/docs/content/docker.md index 319d245732abd..f985874d7845b 100644 --- a/docs/content/docker.md +++ b/docs/content/docker.md @@ -45,7 +45,7 @@ on the host. The _FUSE_ driver is a prerequisite for rclone mounting and should be installed on host: ``` -sudo apt-get -y install fuse +sudo apt-get -y install fuse3 ``` Create two directories required by rclone docker plugin: From 77c1031dcda9b79845653ed5adc680b150f1d950 Mon Sep 17 00:00:00 2001 From: huanghaojun Date: Wed, 9 Apr 2025 18:58:09 +0800 Subject: [PATCH 55/73] vfs: fix inefficient directory caching when directory reads are slow Before this change, when querying directories with large datasets, if the query duration exceeded the directory cache expiration time, the cache became invalid by the time results were retrieved. This means every execution of `_readDir` triggers `_readDirFromEntries`, resulting in prolonged processing times. After this change we update the directory time with the time at the end of the query. --- vfs/dir.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vfs/dir.go b/vfs/dir.go index a4a31c4b6891c..a7f5b73bf51f5 100644 --- a/vfs/dir.go +++ b/vfs/dir.go @@ -579,7 +579,7 @@ func (d *Dir) _readDir() error { return err } - d.read = when + d.read = time.Now() d.cleanupTimer.Reset(time.Duration(d.vfs.Opt.DirCacheTime * 2)) return nil From 7fd3d0d1ab209733d49dc40f3a9df81de2347bf5 Mon Sep 17 00:00:00 2001 From: Markus Gerstel Date: Sun, 9 Mar 2025 10:22:01 +0000 Subject: [PATCH 56/73] s3: documentation regression - fixes #8438 We lost a previous documentation fix (#7077) detailing how to restore single objects from AWS S3 Glacier. Also make clearer that rclone provides restore functionality natively. Co-authored-by: danielkrajnik --- backend/s3/s3.go | 2 +- docs/content/s3.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 716b35ffdd225..1124ae6b4363e 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -4976,7 +4976,7 @@ or from INTELLIGENT-TIERING Archive Access / Deep Archive Access tier to the Fre Usage Examples: - rclone backend restore s3:bucket/path/to/object -o priority=PRIORITY -o lifetime=DAYS + rclone backend restore s3:bucket/path/to/ --include /object -o priority=PRIORITY -o lifetime=DAYS rclone backend restore s3:bucket/path/to/directory -o priority=PRIORITY -o lifetime=DAYS rclone backend restore s3:bucket -o priority=PRIORITY -o lifetime=DAYS rclone backend restore s3:bucket/path/to/directory -o priority=PRIORITY diff --git a/docs/content/s3.md b/docs/content/s3.md index 4d96cde940698..d225312c79bbf 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -771,7 +771,8 @@ tries to access data from the glacier storage class you will see an error like b 2017/09/11 19:07:43 Failed to sync: failed to open source object: Object in GLACIER, restore first: path/to/file In this case you need to [restore](http://docs.aws.amazon.com/AmazonS3/latest/user-guide/restore-archived-objects.html) -the object(s) in question before using rclone. +the object(s) in question before accessing object contents. +The [restore](#restore) section below shows how to do this with rclone. Note that rclone only speaks the S3 API it does not speak the Glacier Vault API, so rclone cannot directly access Glacier Vaults. From a49ccddb81d1745df1710f3ef19251e2f01279df Mon Sep 17 00:00:00 2001 From: simwai <16225108+simwai@users.noreply.github.com> Date: Sat, 29 Mar 2025 21:31:32 +0100 Subject: [PATCH 57/73] cmd/authorize: show required arguments in help text --- cmd/authorize/authorize.go | 8 ++++++-- cmd/authorize/authorize_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 cmd/authorize/authorize_test.go diff --git a/cmd/authorize/authorize.go b/cmd/authorize/authorize.go index ded80186aa603..c4137a4ccb30e 100644 --- a/cmd/authorize/authorize.go +++ b/cmd/authorize/authorize.go @@ -23,19 +23,23 @@ func init() { } var commandDefinition = &cobra.Command{ - Use: "authorize", + Use: "authorize [base64_json_blob | client_id client_secret]", Short: `Remote authorization.`, Long: `Remote authorization. Used to authorize a remote or headless rclone from a machine with a browser - use as instructed by rclone config. +The command requires 1-3 arguments: + - fs name (e.g., "drive", "s3", etc.) + - Either a base64 encoded JSON blob obtained from a previous rclone config session + - Or a client_id and client_secret pair obtained from the remote service + Use --auth-no-open-browser to prevent rclone to open auth link in default browser automatically. Use --template to generate HTML output via a custom Go template. If a blank string is provided as an argument to this flag, the default template is used.`, Annotations: map[string]string{ "versionIntroduced": "v1.27", - // "groups": "", }, RunE: func(command *cobra.Command, args []string) error { cmd.CheckArgs(1, 3, command, args) diff --git a/cmd/authorize/authorize_test.go b/cmd/authorize/authorize_test.go new file mode 100644 index 0000000000000..364da5ed11e4b --- /dev/null +++ b/cmd/authorize/authorize_test.go @@ -0,0 +1,32 @@ +package authorize + +import ( + "bytes" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +func TestAuthorizeCommand(t *testing.T) { + // Test that the Use string is correctly formatted + if commandDefinition.Use != "authorize [base64_json_blob | client_id client_secret]" { + t.Errorf("Command Use string doesn't match expected format: %s", commandDefinition.Use) + } + + // Test that help output contains the argument information + buf := &bytes.Buffer{} + cmd := &cobra.Command{} + cmd.AddCommand(commandDefinition) + cmd.SetOut(buf) + cmd.SetArgs([]string{"authorize", "--help"}) + err := cmd.Execute() + if err != nil { + t.Fatalf("Failed to execute help command: %v", err) + } + + helpOutput := buf.String() + if !strings.Contains(helpOutput, "authorize ") { + t.Errorf("Help output doesn't contain correct usage information") + } +} From 4c3737014a2310952fbf2f8f2f8e79cbcd7c7a10 Mon Sep 17 00:00:00 2001 From: Ben Alex Date: Wed, 9 Apr 2025 11:58:37 +1000 Subject: [PATCH 58/73] iclouddrive: fix so created files are writable At present any created file (eg through the touch command, copy, mount etc) is read-only in iCloud. This has been reported by users at https://forum.rclone.org/t/icloud-and-file-editing-permissions/50659. --- backend/iclouddrive/api/drive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/iclouddrive/api/drive.go b/backend/iclouddrive/api/drive.go index 5d735ebbdeb21..0df9e2ae4335c 100644 --- a/backend/iclouddrive/api/drive.go +++ b/backend/iclouddrive/api/drive.go @@ -631,7 +631,7 @@ func NewUpdateFileInfo() UpdateFileInfo { FileFlags: FileFlags{ IsExecutable: true, IsHidden: false, - IsWritable: false, + IsWritable: true, }, } } From bbceb84cad0b8d484e46ccac7a49fd92c05898cc Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 26 Mar 2025 11:55:17 +0000 Subject: [PATCH 59/73] onedrive: fix metadata ordering in permissions Before this change, due to a quirk in Graph, User permissions could be lost when applying permissions. Fixes #8465 --- backend/onedrive/metadata.go | 50 ++++++++++++ backend/onedrive/metadata_test.go | 125 ++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 backend/onedrive/metadata_test.go diff --git a/backend/onedrive/metadata.go b/backend/onedrive/metadata.go index 12ba522f67565..f62c0702820aa 100644 --- a/backend/onedrive/metadata.go +++ b/backend/onedrive/metadata.go @@ -396,10 +396,57 @@ func (m *Metadata) WritePermissions(ctx context.Context) (err error) { return nil } +// Order the permissions so that any with users come first. +// +// This is to work around a quirk with Graph: +// +// 1. You are adding permissions for both a group and a user. +// 2. The user is a member of the group. +// 3. The permissions for the group and user are the same. +// 4. You are adding the group permission before the user permission. +// +// When all of the above are true, Graph indicates it has added the +// user permission, but it immediately drops it +// +// See: https://github.com/rclone/rclone/issues/8465 +func (m *Metadata) orderPermissions(xs []*api.PermissionsType) { + // Return true if identity has any user permissions + hasUserIdentity := func(identity *api.IdentitySet) bool { + if identity == nil { + return false + } + return identity.User.ID != "" || identity.User.DisplayName != "" || identity.User.Email != "" || identity.User.LoginName != "" + } + // Return true if p has any user permissions + hasUser := func(p *api.PermissionsType) bool { + if hasUserIdentity(p.GetGrantedTo(m.fs.driveType)) { + return true + } + for _, identity := range p.GetGrantedToIdentities(m.fs.driveType) { + if hasUserIdentity(identity) { + return true + } + } + return false + } + // Put Permissions with a user first, leaving unsorted otherwise + slices.SortStableFunc(xs, func(a, b *api.PermissionsType) int { + aHasUser := hasUser(a) + bHasUser := hasUser(b) + if aHasUser && !bHasUser { + return -1 + } else if !aHasUser && bHasUser { + return 1 + } + return 0 + }) +} + // sortPermissions sorts the permissions (to be written) into add, update, and remove queues func (m *Metadata) sortPermissions() (add, update, remove []*api.PermissionsType) { new, old := m.queuedPermissions, m.permissions if len(old) == 0 || m.permsAddOnly { + m.orderPermissions(new) return new, nil, nil // they must all be "add" } @@ -447,6 +494,9 @@ func (m *Metadata) sortPermissions() (add, update, remove []*api.PermissionsType remove = append(remove, o) } } + m.orderPermissions(add) + m.orderPermissions(update) + m.orderPermissions(remove) return add, update, remove } diff --git a/backend/onedrive/metadata_test.go b/backend/onedrive/metadata_test.go new file mode 100644 index 0000000000000..bc74152338730 --- /dev/null +++ b/backend/onedrive/metadata_test.go @@ -0,0 +1,125 @@ +package onedrive + +import ( + "encoding/json" + "testing" + + "github.com/rclone/rclone/backend/onedrive/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOrderPermissions(t *testing.T) { + tests := []struct { + name string + input []*api.PermissionsType + expected []string + }{ + { + name: "empty", + input: []*api.PermissionsType{}, + expected: []string(nil), + }, + { + name: "users first, then group, then none", + input: []*api.PermissionsType{ + {ID: "1", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group1"}}}, + {ID: "2", GrantedToIdentities: []*api.IdentitySet{{User: api.Identity{DisplayName: "Alice"}}}}, + {ID: "3", GrantedTo: &api.IdentitySet{User: api.Identity{DisplayName: "Alice"}}}, + {ID: "4"}, + }, + expected: []string{"2", "3", "1", "4"}, + }, + { + name: "same type unsorted", + input: []*api.PermissionsType{ + {ID: "b", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group B"}}}, + {ID: "a", GrantedTo: &api.IdentitySet{Group: api.Identity{DisplayName: "Group A"}}}, + {ID: "c", GrantedToIdentities: []*api.IdentitySet{{Group: api.Identity{DisplayName: "Group A"}}, {User: api.Identity{DisplayName: "Alice"}}}}, + }, + expected: []string{"c", "b", "a"}, + }, + { + name: "all user identities", + input: []*api.PermissionsType{ + {ID: "c", GrantedTo: &api.IdentitySet{User: api.Identity{DisplayName: "Bob"}}}, + {ID: "a", GrantedTo: &api.IdentitySet{User: api.Identity{Email: "alice@example.com"}}}, + {ID: "b", GrantedToIdentities: []*api.IdentitySet{{User: api.Identity{LoginName: "user3"}}}}, + }, + expected: []string{"c", "a", "b"}, + }, + { + name: "no user or group info", + input: []*api.PermissionsType{ + {ID: "z"}, + {ID: "x"}, + {ID: "y"}, + }, + expected: []string{"z", "x", "y"}, + }, + } + + for _, driveType := range []string{driveTypePersonal, driveTypeBusiness} { + t.Run(driveType, func(t *testing.T) { + for _, tt := range tests { + m := &Metadata{fs: &Fs{driveType: driveType}} + t.Run(tt.name, func(t *testing.T) { + if driveType == driveTypeBusiness { + for i := range tt.input { + tt.input[i].GrantedToV2 = tt.input[i].GrantedTo + tt.input[i].GrantedTo = nil + tt.input[i].GrantedToIdentitiesV2 = tt.input[i].GrantedToIdentities + tt.input[i].GrantedToIdentities = nil + } + } + m.orderPermissions(tt.input) + var gotIDs []string + for _, p := range tt.input { + gotIDs = append(gotIDs, p.ID) + } + assert.Equal(t, tt.expected, gotIDs) + }) + } + }) + } +} + +func TestOrderPermissionsJSON(t *testing.T) { + testJSON := `[ + { + "id": "1", + "grantedToV2": { + "group": { + "id": "group@example.com" + } + }, + "roles": [ + "write" + ] + }, + { + "id": "2", + "grantedToV2": { + "user": { + "id": "user@example.com" + } + }, + "roles": [ + "write" + ] + } +]` + + var testPerms []*api.PermissionsType + err := json.Unmarshal([]byte(testJSON), &testPerms) + require.NoError(t, err) + + m := &Metadata{fs: &Fs{driveType: driveTypeBusiness}} + m.orderPermissions(testPerms) + var gotIDs []string + for _, p := range testPerms { + gotIDs = append(gotIDs, p.ID) + } + assert.Equal(t, []string{"2", "1"}, gotIDs) + +} From 4a10491c2a9c859f89836e7bce901742bdcbcab8 Mon Sep 17 00:00:00 2001 From: Andrew Kreimer Date: Tue, 15 Apr 2025 16:42:40 +0300 Subject: [PATCH 60/73] docs: fix typos via codespell There are some types in the changelog. Fix them via codespell. --- docs/content/changelog.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/changelog.md b/docs/content/changelog.md index 0bdf7c317c0fc..38b2f9333b494 100644 --- a/docs/content/changelog.md +++ b/docs/content/changelog.md @@ -60,7 +60,7 @@ description: "Rclone Changelog" * fs: Make `--links` flag global and add new `--local-links` and `--vfs-links` flags (Nick Craig-Wood) * http servers: Disable automatic authentication skipping for unix sockets in http servers (Moises Lima) * This was making it impossible to use unix sockets with an proxy - * This might now cause rclone to need authenticaton where it didn't before + * This might now cause rclone to need authentication where it didn't before * oauthutil: add support for OAuth client credential flow (Martin Hassack, Nick Craig-Wood) * operations: make log messages consistent for mkdir/rmdir at INFO level (Nick Craig-Wood) * rc: Add `relative` to [vfs/queue-set-expiry](/rc/#vfs-queue-set-expiry) (Nick Craig-Wood) @@ -738,7 +738,7 @@ instead of of `--size-only`, when `check` is not available. * Update all dependencies (Nick Craig-Wood) * Refactor version info and icon resource handling on windows (albertony) * doc updates (albertony, alfish2000, asdffdsazqqq, Dimitri Papadopoulos, Herby Gillot, Joda Stößer, Manoj Ghosh, Nick Craig-Wood) - * Implement `--metadata-mapper` to transform metatadata with a user supplied program (Nick Craig-Wood) + * Implement `--metadata-mapper` to transform metadata with a user supplied program (Nick Craig-Wood) * Add `ChunkWriterDoesntSeek` feature flag and set it for b2 (Nick Craig-Wood) * lib/http: Export basic go string functions for use in `--template` (Gabriel Espinoza) * makefile: Use POSIX compatible install arguments (Mina Galić) @@ -853,7 +853,7 @@ instead of of `--size-only`, when `check` is not available. * Fix "fatal error: concurrent map writes" (Nick Craig-Wood) * B2 * Fix multipart upload: corrupted on transfer: sizes differ XXX vs 0 (Nick Craig-Wood) - * Fix locking window when getting mutipart upload URL (Nick Craig-Wood) + * Fix locking window when getting multipart upload URL (Nick Craig-Wood) * Fix server side copies greater than 4GB (Nick Craig-Wood) * Fix chunked streaming uploads (Nick Craig-Wood) * Reduce default `--b2-upload-concurrency` to 4 to reduce memory usage (Nick Craig-Wood) From aedcd565312e5762922de8b9dd3b82f2f1176c05 Mon Sep 17 00:00:00 2001 From: Ed Craig-Wood <138211970+edc-w@users.noreply.github.com> Date: Wed, 16 Apr 2025 10:42:00 +0100 Subject: [PATCH 61/73] Update README.md removed warp as project sponsor --- README.md | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/README.md b/README.md index 1f47ee9149741..f39d7c5a90fa5 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,4 @@ - -
      + [rclone logo](https://rclone.org/#gh-light-mode-only) [rclone logo](https://rclone.org/#gh-dark-mode-only) From 07bb122d474b8df3ec684378e48e09816a38ff1b Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 24 Apr 2025 13:45:04 +0100 Subject: [PATCH 62/73] drive: metadata: fix error when setting copy-requires-writer-permission on a folder This appears not to be allowed, so this fixes the problem by ignoring that metadata for a folder. Fixes #8517 --- backend/drive/drive.go | 4 ++-- backend/drive/metadata.go | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/drive/drive.go b/backend/drive/drive.go index 4ef3b9befd90a..12be6cef956c6 100644 --- a/backend/drive/drive.go +++ b/backend/drive/drive.go @@ -1763,7 +1763,7 @@ func (f *Fs) createDir(ctx context.Context, pathID, leaf string, metadata fs.Met } var updateMetadata updateMetadataFn if len(metadata) > 0 { - updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true) + updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true, true) if err != nil { return nil, fmt.Errorf("create dir: failed to update metadata: %w", err) } @@ -1794,7 +1794,7 @@ func (f *Fs) updateDir(ctx context.Context, dirID string, metadata fs.Metadata) } dirID = actualID(dirID) updateInfo := &drive.File{} - updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true) + updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true, true) if err != nil { return nil, fmt.Errorf("update dir: failed to update metadata from source object: %w", err) } diff --git a/backend/drive/metadata.go b/backend/drive/metadata.go index c8afa7f8dd954..fea18f65a5f3d 100644 --- a/backend/drive/metadata.go +++ b/backend/drive/metadata.go @@ -508,7 +508,7 @@ type updateMetadataFn func(context.Context, *drive.File) error // // It returns a callback which should be called to finish the updates // after the data is uploaded. -func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update bool) (callback updateMetadataFn, err error) { +func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update, isFolder bool) (callback updateMetadataFn, err error) { callbackFns := []updateMetadataFn{} callback = func(ctx context.Context, info *drive.File) error { for _, fn := range callbackFns { @@ -533,7 +533,9 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs } switch k { case "copy-requires-writer-permission": - if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil { + if isFolder { + fs.Debugf(f, "Ignoring %s=%s as can't set on folders", k, v) + } else if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil { return nil, err } case "writers-can-share": @@ -630,7 +632,7 @@ func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, opti if err != nil { return nil, fmt.Errorf("failed to read metadata from source object: %w", err) } - callback, err = f.updateMetadata(ctx, updateInfo, meta, update) + callback, err = f.updateMetadata(ctx, updateInfo, meta, update, false) if err != nil { return nil, fmt.Errorf("failed to update metadata from source object: %w", err) } From b0310c72731c3d7eea88d7c756daac5466c750c2 Mon Sep 17 00:00:00 2001 From: Jugal Kishore Date: Sun, 27 Apr 2025 22:46:43 +0530 Subject: [PATCH 63/73] docs: correct SSL docs anchor link from #ssl-tls to #tls-ssl Fixed the anchor link in the documentation that points to the SSL/TLS section. This change ensures the link directs correctly to the intended section (#tls-ssl) instead of the incorrect #ssl-tls. No functional code changes, documentation only. --- cmd/serve/s3/serve_s3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md index 1c805ab4809c9..03fb1eb9e09ef 100644 --- a/cmd/serve/s3/serve_s3.md +++ b/cmd/serve/s3/serve_s3.md @@ -14,7 +14,7 @@ docs](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html)). access. Please note that some clients may require HTTPS endpoints. See [the -SSL docs](#ssl-tls) for more information. +SSL docs](#tls-ssl) for more information. This command uses the [VFS directory cache](#vfs-virtual-file-system). All the functionality will work with `--vfs-cache-mode off`. Using From 775f3e49f0b03f7931a2115fab86706134811b72 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 26 Feb 2025 11:13:48 +0000 Subject: [PATCH 64/73] build: update golang.org/x/crypto to v0.35.0 to fix CVE-2025-22869 SSH servers which implement file transfer protocols are vulnerable to a denial of service attack from clients which complete the key exchange slowly, or not at all, causing pending content to be read into memory, but never transmitted. This affects users of `rclone serve sftp`. See: https://pkg.go.dev/vuln/GO-2025-3487 --- go.mod | 12 ++++++------ go.sum | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f4d830b8925ec..a07b2b9690fcd 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/rclone/rclone -go 1.21 +go 1.23.0 require ( bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5 @@ -80,12 +80,12 @@ require ( github.com/zeebo/blake3 v0.2.3 go.etcd.io/bbolt v1.3.10 goftp.io/server/v2 v2.0.1 - golang.org/x/crypto v0.31.0 + golang.org/x/crypto v0.35.0 golang.org/x/net v0.33.0 golang.org/x/oauth2 v0.24.0 - golang.org/x/sync v0.10.0 - golang.org/x/sys v0.28.0 - golang.org/x/text v0.21.0 + golang.org/x/sync v0.11.0 + golang.org/x/sys v0.30.0 + golang.org/x/text v0.22.0 golang.org/x/time v0.8.0 google.golang.org/api v0.211.0 gopkg.in/validator.v2 v2.0.1 @@ -228,5 +228,5 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.1 github.com/pkg/xattr v0.4.10 golang.org/x/mobile v0.0.0-20240716161057-1ad2df20a8b6 - golang.org/x/term v0.27.0 + golang.org/x/term v0.29.0 ) diff --git a/go.sum b/go.sum index 06a76c5ef9a23..86d1d565f2857 100644 --- a/go.sum +++ b/go.sum @@ -684,6 +684,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -788,6 +790,7 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -846,6 +849,7 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -858,6 +862,7 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -873,6 +878,7 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 32ea8f6dc823312704118069b5ee7c1afa0dbb65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 17:25:12 +0000 Subject: [PATCH 65/73] build: update golang.org/x/net to 0.36.0. to fix CVE-2025-22869 SSH servers which implement file transfer protocols are vulnerable to a denial of service attack from clients which complete the key exchange slowly, or not at all, causing pending content to be read into memory, but never transmitted. This updates golang.org/x/net to fix the problem. See: https://pkg.go.dev/vuln/GO-2025-3487 See: https://www.cve.org/CVERecord?id=CVE-2025-22869 Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a07b2b9690fcd..a1c804e3dedbd 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,7 @@ require ( go.etcd.io/bbolt v1.3.10 goftp.io/server/v2 v2.0.1 golang.org/x/crypto v0.35.0 - golang.org/x/net v0.33.0 + golang.org/x/net v0.36.0 golang.org/x/oauth2 v0.24.0 golang.org/x/sync v0.11.0 golang.org/x/sys v0.30.0 diff --git a/go.sum b/go.sum index 86d1d565f2857..fe83ae7154e6c 100644 --- a/go.sum +++ b/go.sum @@ -684,7 +684,6 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -768,6 +767,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From 0121dc11d32a71d6ba0b203b2c9c28ce4538f527 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Mar 2025 22:31:22 +0000 Subject: [PATCH 66/73] build: bump github.com/golang-jwt/jwt/v4 from 4.5.1 to 4.5.2 Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.1 to 4.5.2. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.1...v4.5.2) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a1c804e3dedbd..b20e6235982e7 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,7 @@ require ( go.etcd.io/bbolt v1.3.10 goftp.io/server/v2 v2.0.1 golang.org/x/crypto v0.35.0 - golang.org/x/net v0.36.0 + golang.org/x/net v0.33.0 golang.org/x/oauth2 v0.24.0 golang.org/x/sync v0.11.0 golang.org/x/sys v0.30.0 @@ -225,7 +225,7 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.1.3 - github.com/golang-jwt/jwt/v4 v4.5.1 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/pkg/xattr v0.4.10 golang.org/x/mobile v0.0.0-20240716161057-1ad2df20a8b6 golang.org/x/term v0.29.0 diff --git a/go.sum b/go.sum index fe83ae7154e6c..4c5f00d5fc4ee 100644 --- a/go.sum +++ b/go.sum @@ -289,6 +289,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -684,6 +686,7 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -767,8 +770,6 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From c9f3456341f6bf22dc0711cf06b39c6979730164 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 6 Apr 2025 11:46:13 +0100 Subject: [PATCH 67/73] build: update github.com/golang-jwt/jwt/v5 from 5.2.1 to 5.2.2 to fix CVE-2025-30204 Bumps [github.com/golang-jwt/jwt/v5](https://github.com/golang-jwt/jwt) from 5.2.1 to 5.2.2. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v5.2.1...v5.2.2) See: https://github.com/golang-jwt/jwt/security/advisories/GHSA-mh63-6h87-95cp See: https://www.cve.org/CVERecord?id=CVE-2025-30204 Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index b20e6235982e7..e9ce350dece28 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,7 @@ require ( go.etcd.io/bbolt v1.3.10 goftp.io/server/v2 v2.0.1 golang.org/x/crypto v0.35.0 - golang.org/x/net v0.33.0 + golang.org/x/net v0.36.0 golang.org/x/oauth2 v0.24.0 golang.org/x/sync v0.11.0 golang.org/x/sys v0.30.0 @@ -150,7 +150,7 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.14.0 // indirect @@ -225,7 +225,7 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.1.3 - github.com/golang-jwt/jwt/v4 v4.5.2 + github.com/golang-jwt/jwt/v4 v4.5.1 github.com/pkg/xattr v0.4.10 golang.org/x/mobile v0.0.0-20240716161057-1ad2df20a8b6 golang.org/x/term v0.29.0 diff --git a/go.sum b/go.sum index 4c5f00d5fc4ee..9b5618edac671 100644 --- a/go.sum +++ b/go.sum @@ -289,10 +289,10 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= -github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -686,7 +686,6 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -770,6 +769,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From ef975129a8b7e508a5b942b329e07e429a4d5546 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Apr 2025 23:26:06 +0000 Subject: [PATCH 68/73] build: bump golang.org/x/net from 0.36.0 to 0.38.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.36.0 to 0.38.0. - [Commits](https://github.com/golang/net/compare/v0.36.0...v0.38.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.38.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 16 ++++++++-------- go.sum | 37 ++++++++++++++----------------------- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index e9ce350dece28..4291434bfcd2d 100644 --- a/go.mod +++ b/go.mod @@ -80,12 +80,12 @@ require ( github.com/zeebo/blake3 v0.2.3 go.etcd.io/bbolt v1.3.10 goftp.io/server/v2 v2.0.1 - golang.org/x/crypto v0.35.0 - golang.org/x/net v0.36.0 + golang.org/x/crypto v0.36.0 + golang.org/x/net v0.38.0 golang.org/x/oauth2 v0.24.0 - golang.org/x/sync v0.11.0 - golang.org/x/sys v0.30.0 - golang.org/x/text v0.22.0 + golang.org/x/sync v0.12.0 + golang.org/x/sys v0.31.0 + golang.org/x/text v0.23.0 golang.org/x/time v0.8.0 google.golang.org/api v0.211.0 gopkg.in/validator.v2 v2.0.1 @@ -150,7 +150,7 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.14.0 // indirect @@ -225,8 +225,8 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v1.1.3 - github.com/golang-jwt/jwt/v4 v4.5.1 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/pkg/xattr v0.4.10 golang.org/x/mobile v0.0.0-20240716161057-1ad2df20a8b6 - golang.org/x/term v0.29.0 + golang.org/x/term v0.30.0 ) diff --git a/go.sum b/go.sum index 9b5618edac671..923a2d10adfed 100644 --- a/go.sum +++ b/go.sum @@ -287,12 +287,10 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= -github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -684,9 +682,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -767,10 +764,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -791,9 +786,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -850,9 +844,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -863,9 +856,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -879,9 +871,8 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 84f11ae4480986d0cc775bed55219cdeb0844fe7 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 30 Apr 2025 16:13:34 +0100 Subject: [PATCH 69/73] build: stop building with go < go1.23 as security updates forbade it --- .github/workflows/build.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b251e7ec38cd4..b1a81d1d2a9c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22'] + job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os'] include: - job_name: linux @@ -80,18 +80,6 @@ jobs: compile_all: true deploy: true - - job_name: go1.21 - os: ubuntu-latest - go: '1.21' - quicktest: true - racequicktest: true - - - job_name: go1.22 - os: ubuntu-latest - go: '1.22' - quicktest: true - racequicktest: true - name: ${{ matrix.job_name }} runs-on: ${{ matrix.os }} From 3d3ea9ee30ef56d1a80b13cc4244a2a9a218c55d Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Wed, 26 Feb 2025 12:21:35 +0000 Subject: [PATCH 70/73] vfs: fix integration test failures In this commit ceef78ce4493f8d0 vfs: fix directory cache serving stale data We added a new test which caused lots of integration test failures. This fixes the problem by disabling the test unless the feature flag DirModTimeUpdatesOnWrite is present on the remote. --- vfs/dir_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vfs/dir_test.go b/vfs/dir_test.go index 69e45c269cedf..5a4b63e04a8b6 100644 --- a/vfs/dir_test.go +++ b/vfs/dir_test.go @@ -658,10 +658,14 @@ func TestDirFileOpen(t *testing.T) { } func TestDirEntryModTimeInvalidation(t *testing.T) { - if runtime.GOOS == "windows" { + r, vfs := newTestVFS(t) + features := r.Fremote.Features() + if !features.DirModTimeUpdatesOnWrite { + t.Skip("Need DirModTimeUpdatesOnWrite") + } + if features.IsLocal && runtime.GOOS == "windows" { t.Skip("dirent modtime is unreliable on Windows filesystems") } - r, vfs := newTestVFS(t) // Needs to be less than 2x the wait time below, othewrwise the entry // gets cleared out before it had a chance to be updated. From 13875b4500fc77a073fae58a1b771530706b7392 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 1 May 2025 15:33:17 +0100 Subject: [PATCH 71/73] Version v1.69.2 --- MANUAL.html | 113 ++++++++-- MANUAL.md | 120 +++++++--- MANUAL.txt | 165 ++++++++++---- docs/content/changelog.md | 44 ++++ docs/content/commands/rclone.md | 2 +- docs/content/commands/rclone_authorize.md | 7 +- docs/content/commands/rclone_copyto.md | 2 + docs/content/commands/rclone_copyurl.md | 2 +- docs/content/commands/rclone_mount.md | 4 +- docs/content/commands/rclone_nfsmount.md | 4 +- docs/content/commands/rclone_serve_dlna.md | 4 +- docs/content/commands/rclone_serve_docker.md | 4 +- docs/content/commands/rclone_serve_ftp.md | 4 +- docs/content/commands/rclone_serve_http.md | 4 +- docs/content/commands/rclone_serve_nfs.md | 4 +- docs/content/commands/rclone_serve_s3.md | 6 +- docs/content/commands/rclone_serve_sftp.md | 4 +- docs/content/commands/rclone_serve_webdav.md | 4 +- docs/content/flags.md | 2 +- docs/content/s3.md | 2 +- rclone.1 | 221 +++++++++++++++---- 21 files changed, 544 insertions(+), 178 deletions(-) diff --git a/MANUAL.html b/MANUAL.html index 0cbfcecd64d32..471b22e475fbc 100644 --- a/MANUAL.html +++ b/MANUAL.html @@ -5,6 +5,7 @@ + rclone(1) User Manual