KurlesHS/rclone
1
0
Fork 0
forked from TrueCloudLab/rclone
Code Pull requests Activity

operations: fix accounting for multi-thread transfers

Browse source 
This stops the accounting moving in large chunks and gets it to move
as the data is read out of the buffer.
This commit is contained in:
Nick Craig-Wood 2023-08-24 17:16:01 +01:00
parent 2f424ceecf
commit 1b5b36523b
1 changed files with 8 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
fs/operations/multithread.go
View file

@ -91,16 +91,19 @@ func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int, writ
var rs io.ReadSeeker var rs io.ReadSeeker
if mc.noSeek { if mc.noSeek {
// Read directly if we are sure we aren't going to seek // Read directly if we are sure we aren't going to seek
rs = readers.NoSeeker{Reader: rc} // and account with accounting
rs = readers.NoSeeker{Reader: mc.acc.WrapStream(rc)}
} else { } else {
// Read the chunk into buffered reader // Read the chunk into buffered reader
wr := multipart.NewRW() rw := multipart.NewRW()
defer fs.CheckClose(wr, &err) defer fs.CheckClose(rw, &err)
_, err = io.CopyN(wr, rc, size) _, err = io.CopyN(rw, rc, size)
if err != nil { if err != nil {
return fmt.Errorf("multi-thread copy: failed to read chunk: %w", err) return fmt.Errorf("multi-thread copy: failed to read chunk: %w", err)
} }
rs = wr // Account as we go
rw.SetAccounting(mc.acc.AccountRead)
rs = rw
} }
// Write the chunk // Write the chunk
@ -109,13 +112,6 @@ func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int, writ
return fmt.Errorf("multi-thread copy: failed to write chunk: %w", err) return fmt.Errorf("multi-thread copy: failed to write chunk: %w", err)
} }
// FIXME: Wrap ReadSeeker for Accounting
// However, to ensure reporting is correctly seeks have to be handled properly
errAccRead := mc.acc.AccountRead(int(bytesWritten))
if errAccRead != nil {
return errAccRead
}
fs.Debugf(mc.src, "multi-thread copy: stream %d/%d (%d-%d) size %v finished", stream+1, mc.numChunks, start, end, fs.SizeSuffix(bytesWritten)) fs.Debugf(mc.src, "multi-thread copy: stream %d/%d (%d-%d) size %v finished", stream+1, mc.numChunks, start, end, fs.SizeSuffix(bytesWritten))
return nil return nil
} }