Merge pull request #4908 from MichaelEischer/improve-anonymous-s3

S3: Improve anonymous access

changelog/unreleased/issue-4707

@@ -0,0 +1,14 @@
+Change: Disallow S3 anonymous authentication by default
+
+When using the S3 backend with anonymous authentication, it continuously tried
+to retrieve new authentication credentials, which caused bad performance.
+
+Now, to use anonymous authentication, it is necessary to pass the option `-o
+s3.unsafe-anonymous-auth=true` to restic.
+
+It is temporarily possible to revert to the old behavior by setting the
+environment variable `RESTIC_FEATURES=explicit-s3-anonymous-auth=false`. Note
+that this feature flag will be removed in the next minor restic version.
+
+https://github.com/restic/restic/issues/4707
+https://github.com/restic/restic/pull/4908

internal/backend/s3/config.go

@@ -23,11 +23,12 @@ type Config struct {
 	Layout       string `option:"layout" help:"use this backend layout (default: auto-detect) (deprecated)"`
 	StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
 
-	Connections   uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
-	MaxRetries    uint   `option:"retries" help:"set the number of retries attempted"`
-	Region        string `option:"region" help:"set region"`
-	BucketLookup  string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"`
-	ListObjectsV1 bool   `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"`
+	Connections         uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
+	MaxRetries          uint   `option:"retries" help:"set the number of retries attempted"`
+	Region              string `option:"region" help:"set region"`
+	BucketLookup        string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"`
+	ListObjectsV1       bool   `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"`
+	UnsafeAnonymousAuth bool   `option:"unsafe-anonymous-auth" help:"use anonymous authentication"`
 }
 
 // NewConfig returns a new Config with the default values filled in.

internal/backend/s3/s3.go

@@ -52,7 +52,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
 		minio.MaxRetry = int(cfg.MaxRetries)
 	}
 
-	creds, err := getCredentials(cfg)
+	creds, err := getCredentials(cfg, rt)
 	if err != nil {
 		return nil, errors.Wrap(err, "s3.getCredentials")
 	}
@@ -97,7 +97,11 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
 
 // getCredentials -- runs through the various credential types and returns the first one that works.
 // additionally if the user has specified a role to assume, it will do that as well.
-func getCredentials(cfg Config) (*credentials.Credentials, error) {
+func getCredentials(cfg Config, tr http.RoundTripper) (*credentials.Credentials, error) {
+	if cfg.UnsafeAnonymousAuth {
+		return credentials.New(&credentials.Static{}), nil
+	}
+
 	// Chains all credential types, in the following order:
 	// 	- Static credentials provided by user
 	//	- AWS env vars (i.e. AWS_ACCESS_KEY_ID)
@@ -120,7 +124,7 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) {
 		&credentials.FileMinioClient{},
 		&credentials.IAM{
 			Client: &http.Client{
-				Transport: http.DefaultTransport,
+				Transport: tr,
 			},
 		},
 	})
@@ -131,7 +135,15 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) {
 	}
 
 	if c.SignerType == credentials.SignatureAnonymous {
+		// Fail if no credentials were found to prevent repeated attempts to (unsuccessfully) retrieve new credentials.
+		// The first attempt still has to timeout which slows down restic usage considerably. Thus, migrate towards forcing
+		// users to explicitly decide between authenticated and anonymous access.
+		if feature.Flag.Enabled(feature.ExplicitS3AnonymousAuth) {
+			return nil, fmt.Errorf("no credentials found. Use `-o s3.unsafe-anonymous-auth=true` for anonymous authentication")
+		}
+
 		debug.Log("using anonymous access for %#v", cfg.Endpoint)
+		creds = credentials.New(&credentials.Static{})
 	}
 
 	roleArn := os.Getenv("RESTIC_AWS_ASSUME_ROLE_ARN")

internal/feature/registry.go

@@ -9,6 +9,7 @@ const (
 	DeprecateLegacyIndex    FlagName = "deprecate-legacy-index"
 	DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout"
 	DeviceIDForHardlinks    FlagName = "device-id-for-hardlinks"
+	ExplicitS3AnonymousAuth FlagName = "explicit-s3-anonymous-auth"
 	SafeForgetKeepTags      FlagName = "safe-forget-keep-tags"
 )
 
@@ -18,6 +19,7 @@ func init() {
 		DeprecateLegacyIndex:    {Type: Beta, Description: "disable support for index format used by restic 0.1.0. Use `restic repair index` to update the index if necessary."},
 		DeprecateS3LegacyLayout: {Type: Beta, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your S3 repository if necessary."},
 		DeviceIDForHardlinks:    {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"},
+		ExplicitS3AnonymousAuth: {Type: Beta, Description: "forbid anonymous S3 authentication unless `-o s3.unsafe-anonymous-auth=true` is set"},
 		SafeForgetKeepTags:      {Type: Beta, Description: "prevent deleting all snapshots if the tag passed to `forget --keep-tags tagname` does not exist"},
 	})
 }