Skip to content

Commit 8ffde40

Browse files
committed
operations: fix too many connections open when using --max-memory
Before this change we opened the connection before allocating memory. This meant a long wait sometimes for memory and too many connections open. Now we allocate the memory first before opening the connection.
1 parent 117d8d9 commit 8ffde40

1 file changed

Lines changed: 8 additions & 2 deletions

File tree

fs/operations/multithread.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"github.com/rclone/rclone/fs/accounting"
1313
"github.com/rclone/rclone/lib/atexit"
1414
"github.com/rclone/rclone/lib/multipart"
15+
"github.com/rclone/rclone/lib/pool"
1516
"golang.org/x/sync/errgroup"
1617
)
1718

@@ -76,6 +77,13 @@ func (mc *multiThreadCopyState) copyChunk(ctx context.Context, chunk int, writer
7677
end := min(start+mc.partSize, mc.size)
7778
size := end - start
7879

80+
// Reserve the memory first so we don't open the source and wait for memory buffers for ages
81+
var rw *pool.RW
82+
if !mc.noBuffering {
83+
rw = multipart.NewRW().Reserve(size)
84+
defer fs.CheckClose(rw, &err)
85+
}
86+
7987
fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d (%d-%d) size %v starting", chunk+1, mc.numChunks, start, end, fs.SizeSuffix(size))
8088

8189
rc, err := Open(ctx, mc.src, &fs.RangeOption{Start: start, End: end - 1})
@@ -92,8 +100,6 @@ func (mc *multiThreadCopyState) copyChunk(ctx context.Context, chunk int, writer
92100
rs = rc
93101
} else {
94102
// Read the chunk into buffered reader
95-
rw := multipart.NewRW().Reserve(size)
96-
defer fs.CheckClose(rw, &err)
97103
_, err = io.CopyN(rw, rc, size)
98104
if err != nil {
99105
return fmt.Errorf("multi-thread copy: failed to read chunk: %w", err)

0 commit comments

Comments
 (0)