Merge pull request #3085 from LordGaav/s3-list-objects-v1-flag

Extended option to select V1 API for ListObjects on S3 backend

changelog/unreleased/issue-3083

@@ -0,0 +1,18 @@
+Enhancement: Allow usage of deprecated S3 ListObjectsV1 API
+
+Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken
+`ListObjectsV2` implementation which cause problems for restic when using their
+API endpoints. When a broken server implementation is used, restic prints
+errors similar to the following:
+
+    List() returned error: Truncated response should have continuation token set
+
+As a temporary workaround, restic now allows using the older `ListObjects`
+endpoint by setting the `s3.list-objects-v1` extended option, for instance:
+
+    restic -o s3.list-objects-v1=true snapshots
+
+This option may be removed in future versions of restic.
+
+https://github.com/restic/restic/issues/3083
+https://github.com/restic/restic/pull/3085

doc/030_preparing_a_new_repo.rst

@@ -239,6 +239,15 @@ For an S3-compatible server that is not Amazon (like Minio, see below),
 or is only available via HTTP, you can specify the URL to the server
 like this: ``s3:http://server:port/bucket_name``.
 
+
+.. note:: Certain S3-compatible servers do not properly implement the
+          ``ListObjectsV2`` API, most notably Ceph versions before v14.2.5. On these
+          backends, as a temporary workaround, you can provide the
+          ``-o s3.list-objects-v1=true`` option to use the older
+          ``ListObjects`` API instead. This option may be removed in future
+          versions of restic.
+
+
 Minio Server
 ************
 

internal/backend/s3/config.go

@@ -20,16 +20,18 @@ type Config struct {
 	Layout        string `option:"layout" help:"use this backend layout (default: auto-detect)"`
 	StorageClass  string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
 
-	Connections  uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
-	MaxRetries   uint   `option:"retries" help:"set the number of retries attempted"`
-	Region       string `option:"region" help:"set region"`
-	BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."`
+	Connections   uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+	MaxRetries    uint   `option:"retries" help:"set the number of retries attempted"`
+	Region        string `option:"region" help:"set region"`
+	BucketLookup  string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"`
+	ListObjectsV1 bool   `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"`
 }
 
 // NewConfig returns a new Config with the default values filled in.
 func NewConfig() Config {
 	return Config{
-		Connections: 5,
+		Connections:   5,
+		ListObjectsV1: false,
 	}
 }
 

internal/backend/s3/s3.go

@@ -205,9 +205,12 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo,
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1)
+
 	for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{
 		Prefix:    dir,
 		Recursive: false,
+		UseV1:     be.cfg.ListObjectsV1,
 	}) {
 		if obj.Err != nil {
 			return nil, err
@@ -427,12 +430,15 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1)
+
 	// NB: unfortunately we can't protect this with be.sem.GetToken() here.
 	// Doing so would enable a deadlock situation (gh-1399), as ListObjects()
 	// starts its own goroutine and returns results via a channel.
 	listresp := be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{
 		Prefix:    prefix,
 		Recursive: recursive,
+		UseV1:     be.cfg.ListObjectsV1,
 	})
 
 	for obj := range listresp {

internal/options/options.go

@@ -199,6 +199,14 @@ func (o Options) Apply(ns string, dst interface{}) error {
 
 			v.Field(i).SetUint(vi)
 
+		case "bool":
+			vi, err := strconv.ParseBool(value)
+			if err != nil {
+				return err
+			}
+
+			v.Field(i).SetBool(vi)
+
 		case "Duration":
 			d, err := time.ParseDuration(value)
 			if err != nil {