rclone/backend/chunker/chunker_internal_test.go

148 lines
5.2 KiB
Go
Raw Normal View History

chunker: implementation + required fstest patch Note: chunker implements many irrelevant methods (UserInfo, Disconnect etc), but they are required by TestIntegration/FsCheckWrap and cannot be removed. Dropped API methods: MergeDirs DirCacheFlush PublicLink UserInfo Disconnect OpenWriterAt Meta formats: - renamed old simplejson format to wdmrcompat. - new simplejson format supports hash sums and verification of chunk size/count. Change list: - split-chunking overlay for mailru - add to all - fix linter errors - fix integration tests - support chunks without meta object - fix package paths - propagate context - fix formatting - implement new required wrapper interfaces - also test large file uploads - simplify options - user friendly name pattern - set default chunk size 2G - fix building with golang 1.9 - fix ci/cd on a separate branch - fix updated object name (SyncUTFNorm failed) - fix panic in Box overlay - workaround: Box rename failed if name taken - enhance comments in unit test - fix formatting - embed wrapped remote rather than inherit - require wrapped remote to support move (or copy) - implement 3 (keep fstest) - drop irrelevant file system interfaces - factor out Object.mainChunk - refactor TestLargeUpload as InternalTest - add unit test for chunk name formats - new improved simplejson meta format - tricky case in test FsIsFile (fix+ignore) - remove debugging print - hide temporary objects from listings - fix bugs in chunking reader: - return EOF immediately when all data is sent - handle case when wrapped remote puts by hash (bug detected by TestRcat) - chunked file hashing (feature) - server-side copy across configs (feature) - robust cleanup of temporary chunks in Put - linear download strategy (no read-ahead, feature) - fix unexpected EOF in the box multipart uploader - throw error if destination ignores data
2019-06-09 17:41:48 +00:00
package chunker
import (
"context"
chunker: implementation + required fstest patch Note: chunker implements many irrelevant methods (UserInfo, Disconnect etc), but they are required by TestIntegration/FsCheckWrap and cannot be removed. Dropped API methods: MergeDirs DirCacheFlush PublicLink UserInfo Disconnect OpenWriterAt Meta formats: - renamed old simplejson format to wdmrcompat. - new simplejson format supports hash sums and verification of chunk size/count. Change list: - split-chunking overlay for mailru - add to all - fix linter errors - fix integration tests - support chunks without meta object - fix package paths - propagate context - fix formatting - implement new required wrapper interfaces - also test large file uploads - simplify options - user friendly name pattern - set default chunk size 2G - fix building with golang 1.9 - fix ci/cd on a separate branch - fix updated object name (SyncUTFNorm failed) - fix panic in Box overlay - workaround: Box rename failed if name taken - enhance comments in unit test - fix formatting - embed wrapped remote rather than inherit - require wrapped remote to support move (or copy) - implement 3 (keep fstest) - drop irrelevant file system interfaces - factor out Object.mainChunk - refactor TestLargeUpload as InternalTest - add unit test for chunk name formats - new improved simplejson meta format - tricky case in test FsIsFile (fix+ignore) - remove debugging print - hide temporary objects from listings - fix bugs in chunking reader: - return EOF immediately when all data is sent - handle case when wrapped remote puts by hash (bug detected by TestRcat) - chunked file hashing (feature) - server-side copy across configs (feature) - robust cleanup of temporary chunks in Put - linear download strategy (no read-ahead, feature) - fix unexpected EOF in the box multipart uploader - throw error if destination ignores data
2019-06-09 17:41:48 +00:00
"flag"
"fmt"
"testing"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/fstests"
"github.com/stretchr/testify/assert"
)
// Command line flags
var (
UploadKilobytes = flag.Int("upload-kilobytes", 0, "Upload size in Kilobytes, set this to test large uploads")
)
// test that chunking does not break large uploads
func (f *Fs) InternalTestPutLarge(t *testing.T, kilobytes int) {
t.Run(fmt.Sprintf("PutLarge%dk", kilobytes), func(t *testing.T) {
fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
chunker: implementation + required fstest patch Note: chunker implements many irrelevant methods (UserInfo, Disconnect etc), but they are required by TestIntegration/FsCheckWrap and cannot be removed. Dropped API methods: MergeDirs DirCacheFlush PublicLink UserInfo Disconnect OpenWriterAt Meta formats: - renamed old simplejson format to wdmrcompat. - new simplejson format supports hash sums and verification of chunk size/count. Change list: - split-chunking overlay for mailru - add to all - fix linter errors - fix integration tests - support chunks without meta object - fix package paths - propagate context - fix formatting - implement new required wrapper interfaces - also test large file uploads - simplify options - user friendly name pattern - set default chunk size 2G - fix building with golang 1.9 - fix ci/cd on a separate branch - fix updated object name (SyncUTFNorm failed) - fix panic in Box overlay - workaround: Box rename failed if name taken - enhance comments in unit test - fix formatting - embed wrapped remote rather than inherit - require wrapped remote to support move (or copy) - implement 3 (keep fstest) - drop irrelevant file system interfaces - factor out Object.mainChunk - refactor TestLargeUpload as InternalTest - add unit test for chunk name formats - new improved simplejson meta format - tricky case in test FsIsFile (fix+ignore) - remove debugging print - hide temporary objects from listings - fix bugs in chunking reader: - return EOF immediately when all data is sent - handle case when wrapped remote puts by hash (bug detected by TestRcat) - chunked file hashing (feature) - server-side copy across configs (feature) - robust cleanup of temporary chunks in Put - linear download strategy (no read-ahead, feature) - fix unexpected EOF in the box multipart uploader - throw error if destination ignores data
2019-06-09 17:41:48 +00:00
ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
Path: fmt.Sprintf("chunker-upload-%dk", kilobytes),
Size: int64(kilobytes) * int64(fs.KibiByte),
})
})
}
func (f *Fs) InternalTestChunkNameFormat(t *testing.T) {
savedNameFormat := f.opt.NameFormat
savedStartFrom := f.opt.StartFrom
defer func() {
// restore original settings
_ = f.parseNameFormat(savedNameFormat)
f.opt.StartFrom = savedStartFrom
}()
var err error
err = f.parseNameFormat("*.rclone_chunk.###")
assert.NoError(t, err)
assert.Equal(t, `%s.rclone_chunk.%03d`, f.nameFormat)
assert.Equal(t, `^(.+)\.rclone_chunk\.([0-9]{3,})$`, f.nameRegexp.String())
err = f.parseNameFormat("*.rclone_chunk.#")
assert.NoError(t, err)
assert.Equal(t, `%s.rclone_chunk.%d`, f.nameFormat)
assert.Equal(t, `^(.+)\.rclone_chunk\.([0-9]+)$`, f.nameRegexp.String())
err = f.parseNameFormat("*_chunk_#####")
assert.NoError(t, err)
assert.Equal(t, `%s_chunk_%05d`, f.nameFormat)
assert.Equal(t, `^(.+)_chunk_([0-9]{5,})$`, f.nameRegexp.String())
err = f.parseNameFormat("*-chunk-#")
assert.NoError(t, err)
assert.Equal(t, `%s-chunk-%d`, f.nameFormat)
assert.Equal(t, `^(.+)-chunk-([0-9]+)$`, f.nameRegexp.String())
err = f.parseNameFormat("_*-chunk-##,")
assert.NoError(t, err)
assert.Equal(t, `_%s-chunk-%02d,`, f.nameFormat)
assert.Equal(t, `^_(.+)-chunk-([0-9]{2,}),$`, f.nameRegexp.String())
err = f.parseNameFormat(`*-chunk-#-%^$()[]{}.+-!?:\/`)
assert.NoError(t, err)
assert.Equal(t, `%s-chunk-%d-%%^$()[]{}.+-!?:\/`, f.nameFormat)
assert.Equal(t, `^(.+)-chunk-([0-9]+)-%\^\$\(\)\[\]\{\}\.\+-!\?:\\/$`, f.nameRegexp.String())
err = f.parseNameFormat("chunk-#")
assert.Error(t, err)
err = f.parseNameFormat("*-chunk")
assert.Error(t, err)
err = f.parseNameFormat("*-*-chunk-#")
assert.Error(t, err)
err = f.parseNameFormat("*-chunk-#-#")
assert.Error(t, err)
err = f.parseNameFormat("#-chunk-*")
assert.Error(t, err)
err = f.parseNameFormat("*#")
assert.NoError(t, err)
err = f.parseNameFormat("**#")
assert.Error(t, err)
err = f.parseNameFormat("#*")
assert.Error(t, err)
err = f.parseNameFormat("")
assert.Error(t, err)
err = f.parseNameFormat("-")
assert.Error(t, err)
f.opt.StartFrom = 2
err = f.parseNameFormat("*.chunk.###")
assert.NoError(t, err)
assert.Equal(t, `%s.chunk.%03d`, f.nameFormat)
assert.Equal(t, `^(.+)\.chunk\.([0-9]{3,})$`, f.nameRegexp.String())
assert.Equal(t, "fish.chunk.003", f.makeChunkName("fish", 1, -1))
assert.Equal(t, "fish.chunk.011..tmp_0000054321", f.makeChunkName("fish", 9, 54321))
assert.Equal(t, "fish.chunk.011..tmp_1234567890", f.makeChunkName("fish", 9, 1234567890))
assert.Equal(t, "fish.chunk.1916..tmp_123456789012345", f.makeChunkName("fish", 1914, 123456789012345))
name, chunkNo, tempNo := f.parseChunkName("fish.chunk.003")
assert.True(t, name == "fish" && chunkNo == 1 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.004..tmp_0000000021")
assert.True(t, name == "fish" && chunkNo == 2 && tempNo == 21)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.021")
assert.True(t, name == "fish" && chunkNo == 19 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.323..tmp_1234567890123456789")
assert.True(t, name == "fish" && chunkNo == 321 && tempNo == 1234567890123456789)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.3")
assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.001")
assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.21")
assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.-21")
assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.004.tmp_0000000021")
assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.003..tmp_123456789")
assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.003..tmp_012345678901234567890123456789")
assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
name, chunkNo, tempNo = f.parseChunkName("fish.chunk.003..tmp_-1")
assert.True(t, name == "" && chunkNo == -1 && tempNo == -1)
}
func (f *Fs) InternalTest(t *testing.T) {
t.Run("PutLarge", func(t *testing.T) {
if *UploadKilobytes <= 0 {
t.Skip("-upload-kilobytes is not set")
}
f.InternalTestPutLarge(t, *UploadKilobytes)
})
t.Run("ChunkNameFormat", func(t *testing.T) {
f.InternalTestChunkNameFormat(t)
})
}
var _ fstests.InternalTester = (*Fs)(nil)