diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go index c4c73c5c3..2ce124a90 100644 --- a/backend/azureblob/azureblob.go +++ b/backend/azureblob/azureblob.go @@ -53,6 +53,11 @@ const ( maxUploadCutoff = 256 * fs.MebiByte defaultAccessTier = azblob.AccessTierNone maxTryTimeout = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing) + // Default storage account, key and blob endpoint for emulator support, + // though it is a base64 key checked in here, it is publicly available secret. + emulatorAccount = "devstoreaccount1" + emulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1" ) // Register with Fs @@ -63,13 +68,17 @@ func init() { NewFs: NewFs, Options: []fs.Option{{ Name: "account", - Help: "Storage Account Name (leave blank to use connection string or SAS URL)", + Help: "Storage Account Name (leave blank to use connection string or SAS URL or Emulator)", }, { Name: "key", - Help: "Storage Account Key (leave blank to use connection string or SAS URL)", + Help: "Storage Account Key (leave blank to use connection string or SAS URL or Emulator)", }, { Name: "sas_url", - Help: "SAS URL for container level access only\n(leave blank if using account/key or connection string)", + Help: "SAS URL for container level access only\n(leave blank if using account/key or connection string or Emulator)", + }, { + Name: "use_emulator", + Help: "Uses local storage emulator if provided as 'true' (leave blank if using real azure storage endpoint)", + Default: false, }, { Name: "endpoint", Help: "Endpoint for the service\nLeave blank normally.", @@ -129,6 +138,7 @@ type Options struct { ChunkSize fs.SizeSuffix `config:"chunk_size"` ListChunkSize uint `config:"list_chunk"` AccessTier string `config:"access_tier"` + UseEmulator bool `config:"use_emulator"` } // Fs represents a remote azure server @@ -366,6 +376,18 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) { containerURL azblob.ContainerURL ) switch { + case opt.UseEmulator: + credential, err := azblob.NewSharedKeyCredential(emulatorAccount, emulatorAccountKey) + if err != nil { + return nil, errors.Wrapf(err, "Failed to parse credentials") + } + u, err = url.Parse(emulatorBlobEndpoint) + if err != nil { + return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint") + } + pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}}) + serviceURL = azblob.NewServiceURL(*u, pipeline) + containerURL = serviceURL.NewContainerURL(container) case opt.Account != "" && opt.Key != "": credential, err := azblob.NewSharedKeyCredential(opt.Account, opt.Key) if err != nil { diff --git a/docs/content/azureblob.md b/docs/content/azureblob.md index 740239b5b..3416bf270 100644 --- a/docs/content/azureblob.md +++ b/docs/content/azureblob.md @@ -279,3 +279,9 @@ tiering blob to "Hot" or "Cool". MD5 sums are only uploaded with chunked files if the source has an MD5 sum. This will always be the case for a local to azure copy. + +### Azure Storage Emulator Support ### +You can test rlcone with storage emulator locally, to do this make sure azure storage emulator +installed locally and set up a new remote with `rclone config` follow instructions described in +introduction, set `use_emulator` config as `true`, you do not need to provide default account name +or key if using emulator.