swift: implement --swift-use-segments-container to allow >5G files on Blomp

This switches between storing chunks in a separate container suffixed
with `_segments` (the default) and a directory in the root
`.file-segments`)

By default the `.file-segments` mode will be auto selected if
`auth_url`s that require it are detected.

If the `.file-segments` mode is in use then rclone will omit that
directory from listings.

See: https://forum.rclone.org/t/blomp-unable-to-upload-5gb-files/42498/
This commit is contained in:
Nick Craig-Wood 2024-03-11 18:33:11 +00:00
parent 8a18c29835
commit 5bf70c68f1
3 changed files with 225 additions and 202 deletions

View file

@ -118,6 +118,9 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
f.opt.NoChunk = false
f.opt.ChunkSize = 1024 * fs.SizeSuffixBase
segmentContainer := f.root + "_segments"
if !f.opt.UseSegmentsContainer.Value {
segmentContainer = f.root
}
defer func() {
//restore config
f.opt.ChunkSize = preConfChunkSize
@ -147,6 +150,9 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
_, _, err = f.c.Object(ctx, f.rootContainer, path)
assert.Equal(t, swift.ObjectNotFound, err)
prefix := path
if !f.opt.UseSegmentsContainer.Value {
prefix = segmentsDirectory + "/" + prefix
}
objs, err := f.c.Objects(ctx, segmentContainer, &swift.ObjectsOpts{
Prefix: prefix,
})