9cc8ff4dd4
Some storage providers e.g. S3 don't have an efficient rename operation. Before this change, when chunker finished an upload, the server-side copy and delete operations that renamed temporary chunks to their final names could take a significant amount of time. This PR records transaction identifier (versioning) in the metadata of chunker composite objects striving to remove the need for rename operations on such backends. This approach will be triggered be the new "transactions" configuration option, which can be "rename" (the default) or "norename". We implement the new approach for uploads (Put operations). The chunker Move operation still uses the rename operation of underlying backend. Filling this gap is left for a later PR. Co-authored-by: Ivan Andreev <ivandeex@gmail.com>
325 lines
8.3 KiB
YAML
325 lines
8.3 KiB
YAML
tests:
|
|
- path: backend
|
|
addbackend: true
|
|
nobinary: true
|
|
short: true
|
|
- path: fs/operations
|
|
fastlist: true
|
|
- path: fs/sync
|
|
fastlist: true
|
|
- path: vfs
|
|
- path: cmd/serve/restic
|
|
localonly: true
|
|
backends:
|
|
# - backend: "amazonclouddrive"
|
|
# remote: "TestAmazonCloudDrive:"
|
|
# fastlist: false
|
|
- backend: "local"
|
|
remote: ""
|
|
fastlist: false
|
|
- backend: "b2"
|
|
remote: "TestB2:"
|
|
fastlist: true
|
|
listretries: 5
|
|
- backend: "crypt"
|
|
remote: "TestCryptDrive:"
|
|
fastlist: true
|
|
- backend: "crypt"
|
|
remote: "TestCryptSwift:"
|
|
fastlist: false
|
|
## chunker
|
|
- backend: "chunker"
|
|
remote: "TestChunkerLocal:"
|
|
fastlist: true
|
|
- backend: "chunker"
|
|
remote: "TestChunkerNometaLocal:"
|
|
fastlist: true
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk3bLocal:"
|
|
fastlist: true
|
|
maxfile: 6k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk3bNometaLocal:"
|
|
fastlist: true
|
|
maxfile: 6k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk3bNoRenameLocal:"
|
|
fastlist: true
|
|
maxfile: 6k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerMailru:"
|
|
fastlist: true
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bMailru:"
|
|
fastlist: true
|
|
maxfile: 10k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bYandex:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bBox:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerS3:"
|
|
fastlist: true
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bMD5HashS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bSHA1HashS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerOverCrypt:"
|
|
fastlist: true
|
|
maxfile: 6k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bMD5QuickS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
- backend: "chunker"
|
|
remote: "TestChunkerChunk50bSHA1QuickS3:"
|
|
fastlist: true
|
|
maxfile: 1k
|
|
## end chunker
|
|
## begin compress
|
|
- backend: "compress"
|
|
remote: "TestCompress:"
|
|
fastlist: false
|
|
- backend: "compress"
|
|
remote: "TestCompressSwift:"
|
|
fastlist: false
|
|
- backend: "compress"
|
|
remote: "TestCompressDrive:"
|
|
fastlist: false
|
|
- backend: "compress"
|
|
remote: "TestCompressS3:"
|
|
fastlist: false
|
|
## end compress
|
|
- backend: "drive"
|
|
remote: "TestDrive:"
|
|
fastlist: true
|
|
- backend: "dropbox"
|
|
remote: "TestDropbox:"
|
|
fastlist: false
|
|
- backend: "filefabric"
|
|
remote: "TestFileFabric:"
|
|
fastlist: false
|
|
- backend: "googlecloudstorage"
|
|
remote: "TestGoogleCloudStorage:"
|
|
fastlist: true
|
|
- backend: "googlephotos"
|
|
remote: "TestGooglePhotos:"
|
|
tests:
|
|
- backend
|
|
- backend: "hubic"
|
|
remote: "TestHubic:"
|
|
fastlist: false
|
|
- backend: "jottacloud"
|
|
remote: "TestJottacloud:"
|
|
fastlist: true
|
|
- backend: "memory"
|
|
remote: ":memory:"
|
|
fastlist: true
|
|
- backend: "onedrive"
|
|
remote: "TestOneDrive:"
|
|
fastlist: false
|
|
- backend: "onedrive"
|
|
remote: "TestOneDriveBusiness:"
|
|
fastlist: false
|
|
- backend: "onedrive"
|
|
remote: "TestOneDriveCn:"
|
|
fastlist: false
|
|
- backend: "s3"
|
|
remote: "TestS3:"
|
|
fastlist: true
|
|
- backend: "s3"
|
|
remote: "TestS3Minio:"
|
|
fastlist: true
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/SetTier
|
|
- TestIntegration/FsMkdir/FsEncoding/control_chars
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_LF
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_VT
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
- TestIntegration/FsMkdir/FsEncoding/trailing_LF
|
|
- TestIntegration/FsMkdir/FsEncoding/trailing_VT
|
|
- backend: "s3"
|
|
remote: "TestS3MinioEdge:"
|
|
fastlist: true
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/SetTier
|
|
- backend: "s3"
|
|
remote: "TestS3Wasabi:"
|
|
fastlist: true
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_CR
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_LF
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_HT
|
|
- TestIntegration/FsMkdir/FsEncoding/leading_VT
|
|
- TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0
|
|
# Disabled due to excessive rate limiting at DO which cause the tests never to pass
|
|
# This hits the rate limit as documented here: https://www.digitalocean.com/docs/spaces/#limits
|
|
# 2 COPYs per 5 minutes on any individual object in a Space
|
|
# - backend: "s3"
|
|
# remote: "TestS3DigitalOcean:"
|
|
# fastlist: true
|
|
# ignore:
|
|
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
|
|
# - TestIntegration/FsMkdir/FsPutFiles/SetTier
|
|
# - backend: "s3"
|
|
# remote: "TestS3Ceph:"
|
|
# fastlist: true
|
|
# ignore:
|
|
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
|
|
# - TestIntegration/FsMkdir/FsPutFiles/SetTier
|
|
- backend: "s3"
|
|
remote: "TestS3Alibaba:"
|
|
fastlist: true
|
|
- backend: "sftp"
|
|
remote: "TestSFTPOpenssh:"
|
|
fastlist: false
|
|
- backend: "sftp"
|
|
remote: "TestSFTPRclone:"
|
|
fastlist: false
|
|
- backend: "sugarsync"
|
|
remote: "TestSugarSync:Test"
|
|
fastlist: false
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/PublicLink
|
|
- backend: "swift"
|
|
remote: "TestSwiftAIO:"
|
|
fastlist: true
|
|
- backend: "swift"
|
|
remote: "TestSwift:"
|
|
fastlist: true
|
|
# - backend: "swift"
|
|
# remote: "TestSwiftCeph:"
|
|
# fastlist: true
|
|
# ignore:
|
|
# - TestIntegration/FsMkdir/FsPutFiles/FsCopy
|
|
- backend: "yandex"
|
|
remote: "TestYandex:"
|
|
fastlist: false
|
|
- backend: "ftp"
|
|
remote: "TestFTPProftpd:"
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
fastlist: false
|
|
# - backend: "ftp"
|
|
# remote: "TestFTPVsftpd:"
|
|
# ignore:
|
|
# - TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
# fastlist: false
|
|
- backend: "ftp"
|
|
remote: "TestFTPPureftpd:"
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
fastlist: false
|
|
- backend: "ftp"
|
|
remote: "TestFTPRclone:"
|
|
ignore:
|
|
- "TestMultithreadCopy/{size:131071_streams:2}"
|
|
- "TestMultithreadCopy/{size:131072_streams:2}"
|
|
- "TestMultithreadCopy/{size:131073_streams:2}"
|
|
fastlist: false
|
|
- backend: "box"
|
|
remote: "TestBox:"
|
|
fastlist: false
|
|
- backend: "fichier"
|
|
remote: "TestFichier:"
|
|
fastlist: false
|
|
tests:
|
|
- backend
|
|
- backend: "qingstor"
|
|
remote: "TestQingStor:"
|
|
fastlist: false
|
|
oneonly: true
|
|
cleanup: true
|
|
- backend: "azureblob"
|
|
remote: "TestAzureBlob:"
|
|
fastlist: true
|
|
- backend: "pcloud"
|
|
remote: "TestPcloud:"
|
|
fastlist: false
|
|
- backend: "webdav"
|
|
remote: "TestWebdavNextcloud:"
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
- TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
|
|
fastlist: false
|
|
- backend: "webdav"
|
|
remote: "TestWebdavOwncloud:"
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsEncoding/punctuation
|
|
- TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
|
|
- TestIntegration/FsMkdir/FsPutFiles/FsCopy
|
|
- TestCopyFileCopyDest
|
|
- TestServerSideCopy
|
|
- TestSyncCopyDest
|
|
fastlist: false
|
|
- backend: "webdav"
|
|
remote: "TestWebdavRclone:"
|
|
ignore:
|
|
- TestFileReadAtZeroLength
|
|
fastlist: false
|
|
- backend: "cache"
|
|
remote: "TestCache:"
|
|
fastlist: false
|
|
- backend: "mega"
|
|
remote: "TestMega:"
|
|
fastlist: false
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/PublicLink
|
|
- TestDirRename
|
|
- backend: "opendrive"
|
|
remote: "TestOpenDrive:"
|
|
fastlist: false
|
|
- backend: "union"
|
|
remote: "TestUnion:"
|
|
fastlist: false
|
|
- backend: "koofr"
|
|
remote: "TestKoofr:"
|
|
fastlist: false
|
|
- backend: "premiumizeme"
|
|
remote: "TestPremiumizeMe:"
|
|
fastlist: false
|
|
- backend: "putio"
|
|
remote: "TestPutio:"
|
|
fastlist: false
|
|
- backend: "sharefile"
|
|
remote: "TestSharefile:"
|
|
fastlist: false
|
|
- backend: "mailru"
|
|
remote: "TestMailru:"
|
|
subdir: false
|
|
fastlist: false
|
|
- backend: "seafile"
|
|
remote: "TestSeafileV6:"
|
|
fastlist: false
|
|
ignore:
|
|
- TestIntegration/FsMkdir/FsPutFiles/FsDirMove
|
|
- backend: "seafile"
|
|
remote: "TestSeafile:"
|
|
fastlist: true
|
|
- backend: "seafile"
|
|
remote: "TestSeafileEncrypted:"
|
|
fastlist: true
|
|
- backend: "tardigrade"
|
|
remote: "TestTardigrade:"
|
|
fastlist: true
|
|
- backend: "zoho"
|
|
remote: "TestZoho:"
|
|
fastlist: false
|
|
- backend: "hdfs"
|
|
remote: "TestHdfs:"
|
|
fastlist: false
|
|
ignore:
|
|
- TestSyncUTFNorm
|