From 5ffb536aaed98e383b037563dd15509e71cf7620 Mon Sep 17 00:00:00 2001 From: Erik Kristensen Date: Mon, 18 Sep 2023 12:09:32 -0600 Subject: [PATCH 1/4] feat: support AWS assume role --- .gitignore | 1 + changelog/unreleased/issue-4472 | 14 ++++ doc/040_backup.rst | 4 ++ internal/backend/s3/s3.go | 122 +++++++++++++++++++++++--------- 4 files changed, 108 insertions(+), 33 deletions(-) create mode 100644 changelog/unreleased/issue-4472 diff --git a/.gitignore b/.gitignore index b7201c26b..c8c3aa69a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/.idea /restic /restic.exe /.vagrant diff --git a/changelog/unreleased/issue-4472 b/changelog/unreleased/issue-4472 new file mode 100644 index 000000000..97553f946 --- /dev/null +++ b/changelog/unreleased/issue-4472 @@ -0,0 +1,14 @@ +Enhancement: Allow AWS Assume Role to be used for S3 backend + +Previously only credentials discovered via the Minio Click discovery methods +would be used to authenticate. However there are many circumstances where the +discovered credentials have lower permissions and need to assume a specific role. + +New Environment Variables: + +- RESTIC_AWS_ASSUME_ROLE_ARN +- RESTIC_AWS_ASSUME_ROLE_SESSION_NAME +- RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID +- RESTIC_AWS_ASSUME_ROLE_REGION (if need to override from us-east-1) +- RESTIC_AWS_ASSUME_ROLE_POLICY +- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT \ No newline at end of file diff --git a/doc/040_backup.rst b/doc/040_backup.rst index acafe2694..3de8ef554 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -628,6 +628,10 @@ environment variables. The following lists these environment variables: AWS_DEFAULT_REGION Amazon S3 default region AWS_PROFILE Amazon credentials profile (alternative to specifying key and region) AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials) + RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials + RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption + RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption + RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption AZURE_ACCOUNT_NAME Account name for Azure AZURE_ACCOUNT_KEY Account key for Azure diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 98879d0df..ff81a05d6 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -51,40 +51,9 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro minio.MaxRetry = int(cfg.MaxRetries) } - // Chains all credential types, in the following order: - // - Static credentials provided by user - // - AWS env vars (i.e. AWS_ACCESS_KEY_ID) - // - Minio env vars (i.e. MINIO_ACCESS_KEY) - // - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) - // - Minio creds file (i.e. MINIO_SHARED_CREDENTIALS_FILE or ~/.mc/config.json) - // - IAM profile based credentials. (performs an HTTP - // call to a pre-defined endpoint, only valid inside - // configured ec2 instances) - creds := credentials.NewChainCredentials([]credentials.Provider{ - &credentials.EnvAWS{}, - &credentials.Static{ - Value: credentials.Value{ - AccessKeyID: cfg.KeyID, - SecretAccessKey: cfg.Secret.Unwrap(), - }, - }, - &credentials.EnvMinio{}, - &credentials.FileAWSCredentials{}, - &credentials.FileMinioClient{}, - &credentials.IAM{ - Client: &http.Client{ - Transport: http.DefaultTransport, - }, - }, - }) - - c, err := creds.Get() + creds, err := getCredentials(cfg) if err != nil { - return nil, errors.Wrap(err, "creds.Get") - } - - if c.SignerType == credentials.SignatureAnonymous { - debug.Log("using anonymous access for %#v", cfg.Endpoint) + return nil, errors.Wrap(err, "s3.getCredentials") } options := &minio.Options{ @@ -125,6 +94,93 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro return be, nil } +// getCredentials -- runs through the various credential types and returns the first one that works. +// additionally if the user has specified a role to assume, it will do that as well. +func getCredentials(cfg Config) (*credentials.Credentials, error) { + // Chains all credential types, in the following order: + // - Static credentials provided by user + // - AWS env vars (i.e. AWS_ACCESS_KEY_ID) + // - Minio env vars (i.e. MINIO_ACCESS_KEY) + // - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) + // - Minio creds file (i.e. MINIO_SHARED_CREDENTIALS_FILE or ~/.mc/config.json) + // - IAM profile based credentials. (performs an HTTP + // call to a pre-defined endpoint, only valid inside + // configured ec2 instances) + creds := credentials.NewChainCredentials([]credentials.Provider{ + &credentials.Static{ + Value: credentials.Value{ + AccessKeyID: cfg.KeyID, + SecretAccessKey: cfg.Secret.Unwrap(), + }, + }, + &credentials.EnvAWS{}, + &credentials.EnvMinio{}, + &credentials.FileAWSCredentials{}, + &credentials.FileMinioClient{}, + &credentials.IAM{ + Client: &http.Client{ + Transport: http.DefaultTransport, + }, + }, + }) + + c, err := creds.Get() + if err != nil { + return nil, errors.Wrap(err, "creds.Get") + } + + if c.SignerType == credentials.SignatureAnonymous { + debug.Log("using anonymous access for %#v", cfg.Endpoint) + } + + roleArn := os.Getenv("RESTIC_AWS_ASSUME_ROLE_ARN") + if roleArn != "" { + // use the region provided by the configuration by default + awsRegion := cfg.Region + // allow the region to be overridden if for some reason it is required + if len(os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION")) > 0 { + awsRegion = os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION") + } + + sessionName := os.Getenv("RESTIC_AWS_ASSUME_ROLE_SESSION_NAME") + externalID := os.Getenv("RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID") + policy := os.Getenv("RESTIC_AWS_ASSUME_ROLE_POLICY") + stsEndpoint := os.Getenv("RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT") + + if stsEndpoint == "" { + if len(awsRegion) > 0 { + if strings.HasPrefix(awsRegion, "cn-") { + stsEndpoint = "https://sts." + awsRegion + ".amazonaws.com.cn" + } else { + stsEndpoint = "https://sts." + awsRegion + ".amazonaws.com" + } + } else { + stsEndpoint = "https://sts.amazonaws.com" + } + } + + opts := credentials.STSAssumeRoleOptions{ + RoleARN: roleArn, + AccessKey: c.AccessKeyID, + SecretKey: c.SecretAccessKey, + SessionToken: c.SessionToken, + RoleSessionName: sessionName, + ExternalID: externalID, + Policy: policy, + } + if len(awsRegion) > 0 { + opts.Location = awsRegion + } + + creds, err = credentials.NewSTSAssumeRole(stsEndpoint, opts) + if err != nil { + return nil, errors.Wrap(err, "creds.AssumeRole") + } + } + + return creds, nil +} + // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { From 20cf4777cbf52b4b76777faaf20ceb4ba5a71df5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 21:43:41 +0100 Subject: [PATCH 2/4] s3: check for EnvAWS credentials before Static credentials EnvAWS considers more environment variables, including AWS_SESSION_TOKEN and thus should be checked first. --- internal/backend/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index ff81a05d6..d48813cf2 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -107,13 +107,13 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) { // call to a pre-defined endpoint, only valid inside // configured ec2 instances) creds := credentials.NewChainCredentials([]credentials.Provider{ + &credentials.EnvAWS{}, &credentials.Static{ Value: credentials.Value{ AccessKeyID: cfg.KeyID, SecretAccessKey: cfg.Secret.Unwrap(), }, }, - &credentials.EnvAWS{}, &credentials.EnvMinio{}, &credentials.FileAWSCredentials{}, &credentials.FileMinioClient{}, From 02bc73f5eb2ed392aaffa510fbe1cfd392711d28 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 21:44:53 +0100 Subject: [PATCH 3/4] s3: minor code cleanups --- internal/backend/s3/s3.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index d48813cf2..f0447224f 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -138,7 +138,7 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) { // use the region provided by the configuration by default awsRegion := cfg.Region // allow the region to be overridden if for some reason it is required - if len(os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION")) > 0 { + if os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION") != "" { awsRegion = os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION") } @@ -148,7 +148,7 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) { stsEndpoint := os.Getenv("RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT") if stsEndpoint == "" { - if len(awsRegion) > 0 { + if awsRegion != "" { if strings.HasPrefix(awsRegion, "cn-") { stsEndpoint = "https://sts." + awsRegion + ".amazonaws.com.cn" } else { @@ -167,9 +167,7 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) { RoleSessionName: sessionName, ExternalID: externalID, Policy: policy, - } - if len(awsRegion) > 0 { - opts.Location = awsRegion + Location: awsRegion, } creds, err = credentials.NewSTSAssumeRole(stsEndpoint, opts) From 4248c6c3cab07b74a59f7813db75d2d52a59ca6a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 21:45:09 +0100 Subject: [PATCH 4/4] s3: update documentation --- changelog/unreleased/issue-4472 | 16 ++++++++++------ doc/040_backup.rst | 4 +++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/changelog/unreleased/issue-4472 b/changelog/unreleased/issue-4472 index 97553f946..3049fdf30 100644 --- a/changelog/unreleased/issue-4472 +++ b/changelog/unreleased/issue-4472 @@ -1,14 +1,18 @@ Enhancement: Allow AWS Assume Role to be used for S3 backend -Previously only credentials discovered via the Minio Click discovery methods -would be used to authenticate. However there are many circumstances where the -discovered credentials have lower permissions and need to assume a specific role. +Previously only credentials discovered via the Minio discovery methods +were used to authenticate. -New Environment Variables: +However, there are many circumstances where the discovered credentials have +lower permissions and need to assume a specific role. This is now possible +using the following new environment variables. - RESTIC_AWS_ASSUME_ROLE_ARN - RESTIC_AWS_ASSUME_ROLE_SESSION_NAME - RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID -- RESTIC_AWS_ASSUME_ROLE_REGION (if need to override from us-east-1) +- RESTIC_AWS_ASSUME_ROLE_REGION (defaults to us-east-1) - RESTIC_AWS_ASSUME_ROLE_POLICY -- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT \ No newline at end of file +- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT + +https://github.com/restic/restic/issues/4472 +https://github.com/restic/restic/pull/4474 diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 3de8ef554..d36986441 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -631,7 +631,9 @@ environment variables. The following lists these environment variables: RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption - RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption + RESTIC_AWS_ASSUME_ROLE_POLICY Inline Amazion IAM session policy + RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption (default: us-east-1) + RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT URL to the STS endpoint (default is determined based on RESTIC_AWS_ASSUME_ROLE_REGION). You generally do not need to set this, advanced use only. AZURE_ACCOUNT_NAME Account name for Azure AZURE_ACCOUNT_KEY Account key for Azure