diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index 8634d33f9..bffc2f550 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -10,8 +10,10 @@ import ( "crypto/md5" "encoding/base64" "encoding/hex" + "encoding/json" "fmt" "io" + "io/ioutil" "net/http" "net/url" "path" @@ -21,6 +23,7 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/go-autorest/autorest/adal" "github.com/pkg/errors" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" @@ -33,6 +36,7 @@ import ( "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/bucket" "github.com/rclone/rclone/lib/encoder" + "github.com/rclone/rclone/lib/env" "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/pool" "github.com/rclone/rclone/lib/readers" @@ -74,6 +78,20 @@ func init() { Options: []fs.Option{{ Name: "account", Help: "Storage Account Name (leave blank to use SAS URL or Emulator)", + }, { + Name: "service_principal_file", + Help: `Path to file containing credentials for use with a service principal. + +Leave blank normally. Needed only if you want to use a service principal instead of interactive login. + + $ az sp create-for-rbac --name "" \ + --role "Storage Blob Data Owner" \ + --scopes "/subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts//blobServices/default/containers/" \ + > azure-principal.json + +See [Use Azure CLI to assign an Azure role for access to blob and queue data](https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli) +for more details. +`, }, { Name: "key", Help: "Storage Account Key (leave blank to use SAS URL or Emulator)", @@ -167,19 +185,20 @@ This option controls how often unused buffers will be removed from the pool.`, // Options defines the configuration for this backend type Options struct { - Account string `config:"account"` - Key string `config:"key"` - Endpoint string `config:"endpoint"` - SASURL string `config:"sas_url"` - UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` - ChunkSize fs.SizeSuffix `config:"chunk_size"` - ListChunkSize uint `config:"list_chunk"` - AccessTier string `config:"access_tier"` - UseEmulator bool `config:"use_emulator"` - DisableCheckSum bool `config:"disable_checksum"` - MemoryPoolFlushTime fs.Duration `config:"memory_pool_flush_time"` - MemoryPoolUseMmap bool `config:"memory_pool_use_mmap"` - Enc encoder.MultiEncoder `config:"encoding"` + Account string `config:"account"` + ServicePrincipalFile string `config:"service_principal_file"` + Key string `config:"key"` + Endpoint string `config:"endpoint"` + SASURL string `config:"sas_url"` + UploadCutoff fs.SizeSuffix `config:"upload_cutoff"` + ChunkSize fs.SizeSuffix `config:"chunk_size"` + ListChunkSize uint `config:"list_chunk"` + AccessTier string `config:"access_tier"` + UseEmulator bool `config:"use_emulator"` + DisableCheckSum bool `config:"disable_checksum"` + MemoryPoolFlushTime fs.Duration `config:"memory_pool_flush_time"` + MemoryPoolUseMmap bool `config:"memory_pool_use_mmap"` + Enc encoder.MultiEncoder `config:"encoding"` } // Fs represents a remote azure server @@ -354,6 +373,50 @@ func httpClientFactory(client *http.Client) pipeline.Factory { }) } +type servicePrincipalCredentials struct { + AppID string `json:"appId"` + Password string `json:"password"` + Tenant string `json:"tenant"` +} + +const azureActiveDirectoryEndpoint = "https://login.microsoftonline.com/" +const azureStorageEndpoint = "https://storage.azure.com/" + +// newServicePrincipalTokenRefresher takes the client ID and secret, and returns a refresh-able access token. +func newServicePrincipalTokenRefresher(ctx context.Context, credentialsData []byte) (azblob.TokenRefresher, error) { + var spCredentials servicePrincipalCredentials + if err := json.Unmarshal(credentialsData, &spCredentials); err != nil { + return nil, errors.Wrap(err, "error parsing credentials from JSON file") + } + oauthConfig, err := adal.NewOAuthConfig(azureActiveDirectoryEndpoint, spCredentials.Tenant) + if err != nil { + return nil, errors.Wrap(err, "error creating oauth config") + } + + // Create service principal token for Azure Storage. + servicePrincipalToken, err := adal.NewServicePrincipalToken( + *oauthConfig, + spCredentials.AppID, + spCredentials.Password, + azureStorageEndpoint) + if err != nil { + return nil, errors.Wrap(err, "error creating service principal token") + } + + // Wrap token inside a refresher closure. + var tokenRefresher azblob.TokenRefresher = func(credential azblob.TokenCredential) time.Duration { + if err := servicePrincipalToken.Refresh(); err != nil { + panic(err) + } + refreshedToken := servicePrincipalToken.Token() + credential.SetToken(refreshedToken.AccessToken) + exp := refreshedToken.Expires().Sub(time.Now().Add(2 * time.Minute)) + return exp + } + + return tokenRefresher, nil +} + // newPipeline creates a Pipeline using the specified credentials and options. // // this code was copied from azblob.NewPipeline @@ -484,8 +547,27 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e } else { serviceURL = azblob.NewServiceURL(*u, pipeline) } + case opt.ServicePrincipalFile != "": + // Create a standard URL. + u, err = url.Parse(fmt.Sprintf("https://%s.%s", opt.Account, opt.Endpoint)) + if err != nil { + return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint") + } + // Try loading service principal credentials from file. + loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServicePrincipalFile)) + if err != nil { + return nil, errors.Wrap(err, "error opening service principal credentials file") + } + // Create a token refresher from service principal credentials. + tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, loadedCreds) + if err != nil { + return nil, errors.Wrap(err, "failed to create a service principal token") + } + options := azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}} + pipe := f.newPipeline(azblob.NewTokenCredential("", tokenRefresher), options) + serviceURL = azblob.NewServiceURL(*u, pipe) default: - return nil, errors.New("Need account+key or connectionString or sasURL") + return nil, errors.New("Need account+key or connectionString or sasURL or servicePrincipalFile") } f.svcURL = &serviceURL diff --git a/backend/azureblob/azureblob_test.go b/backend/azureblob/azureblob_test.go index 22496e817..4d5930cdd 100644 --- a/backend/azureblob/azureblob_test.go +++ b/backend/azureblob/azureblob_test.go @@ -5,10 +5,12 @@ package azureblob import ( + "context" "testing" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fstest/fstests" + "github.com/stretchr/testify/assert" ) // TestIntegration runs integration tests against the remote @@ -35,3 +37,33 @@ var ( _ fstests.SetUploadChunkSizer = (*Fs)(nil) _ fstests.SetUploadCutoffer = (*Fs)(nil) ) + +// TestServicePrincipalFileSuccess checks that, given a proper JSON file, we can create a token. +func TestServicePrincipalFileSuccess(t *testing.T) { + ctx := context.TODO() + credentials := ` +{ + "appId": "my application (client) ID", + "password": "my secret", + "tenant": "my active directory tenant ID" +} +` + tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials)) + if assert.NoError(t, err) { + assert.NotNil(t, tokenRefresher) + } +} + +// TestServicePrincipalFileFailure checks that, given a JSON file with a missing secret, it returns an error. +func TestServicePrincipalFileFailure(t *testing.T) { + ctx := context.TODO() + credentials := ` +{ + "appId": "my application (client) ID", + "tenant": "my active directory tenant ID" +} +` + _, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials)) + assert.Error(t, err) + assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty") +} diff --git a/go.mod b/go.mod index bcb2a54a8..d10b4afc8 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( cloud.google.com/go v0.70.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.11.0 + github.com/Azure/go-autorest/autorest/adal v0.9.8 github.com/Unknwon/goconfig v0.0.0-20200908083735-df7de6a44db8 github.com/a8m/tree v0.0.0-20201026183218-fce18e2a750e github.com/aalpar/deheap v0.0.0-20200318053559-9a0c2883bd56 diff --git a/go.sum b/go.sum index 015e51262..f63e0ffb3 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest/adal v0.9.2 h1:Aze/GQeAN1RRbGmnUJvUj+tFGBzFdIg3293/A9rbxC4= github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= +github.com/Azure/go-autorest/autorest/adal v0.9.8 h1:bW6ZdxqMYWsxGikpM62SSE3jnvOXVu9SXzJTuj1WM3Y= +github.com/Azure/go-autorest/autorest/adal v0.9.8/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= @@ -176,6 +178,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -660,6 +664,7 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=