distribution/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go
Ahmet Alp Balkan 2ab25288a2
Update vendored azure-sdk-for-go
Updating to a recent version of Azure Storage SDK to be
able to patch some memory leaks through configurable HTTP client
changes which were made possible by recent patches to it.

Signed-off-by: Ahmet Alp Balkan <ahmetalpbalkan@gmail.com>
2016-11-03 13:24:57 -07:00

551 lines
15 KiB
Go

// Package storage provides clients for Microsoft Azure Storage Services.
package storage
import (
"bytes"
"encoding/base64"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
)
const (
// DefaultBaseURL is the domain name used for storage requests when a
// default client is created.
DefaultBaseURL = "core.windows.net"
// DefaultAPIVersion is the Azure Storage API version string used when a
// basic client is created.
DefaultAPIVersion = "2015-02-21"
defaultUseHTTPS = true
// StorageEmulatorAccountName is the fixed storage account used by Azure Storage Emulator
StorageEmulatorAccountName = "devstoreaccount1"
// StorageEmulatorAccountKey is the the fixed storage account used by Azure Storage Emulator
StorageEmulatorAccountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
blobServiceName = "blob"
tableServiceName = "table"
queueServiceName = "queue"
fileServiceName = "file"
storageEmulatorBlob = "127.0.0.1:10000"
storageEmulatorTable = "127.0.0.1:10002"
storageEmulatorQueue = "127.0.0.1:10001"
)
// Client is the object that needs to be constructed to perform
// operations on the storage account.
type Client struct {
// HTTPClient is the http.Client used to initiate API
// requests. If it is nil, http.DefaultClient is used.
HTTPClient *http.Client
accountName string
accountKey []byte
useHTTPS bool
baseURL string
apiVersion string
}
type storageResponse struct {
statusCode int
headers http.Header
body io.ReadCloser
}
type odataResponse struct {
storageResponse
odata odataErrorMessage
}
// AzureStorageServiceError contains fields of the error response from
// Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx
// Some fields might be specific to certain calls.
type AzureStorageServiceError struct {
Code string `xml:"Code"`
Message string `xml:"Message"`
AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"`
QueryParameterName string `xml:"QueryParameterName"`
QueryParameterValue string `xml:"QueryParameterValue"`
Reason string `xml:"Reason"`
StatusCode int
RequestID string
}
type odataErrorMessageMessage struct {
Lang string `json:"lang"`
Value string `json:"value"`
}
type odataErrorMessageInternal struct {
Code string `json:"code"`
Message odataErrorMessageMessage `json:"message"`
}
type odataErrorMessage struct {
Err odataErrorMessageInternal `json:"odata.error"`
}
// UnexpectedStatusCodeError is returned when a storage service responds with neither an error
// nor with an HTTP status code indicating success.
type UnexpectedStatusCodeError struct {
allowed []int
got int
}
func (e UnexpectedStatusCodeError) Error() string {
s := func(i int) string { return fmt.Sprintf("%d %s", i, http.StatusText(i)) }
got := s(e.got)
expected := []string{}
for _, v := range e.allowed {
expected = append(expected, s(v))
}
return fmt.Sprintf("storage: status code from service response is %s; was expecting %s", got, strings.Join(expected, " or "))
}
// Got is the actual status code returned by Azure.
func (e UnexpectedStatusCodeError) Got() int {
return e.got
}
// NewBasicClient constructs a Client with given storage service name and
// key.
func NewBasicClient(accountName, accountKey string) (Client, error) {
if accountName == StorageEmulatorAccountName {
return NewEmulatorClient()
}
return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS)
}
//NewEmulatorClient contructs a Client intended to only work with Azure
//Storage Emulator
func NewEmulatorClient() (Client, error) {
return NewClient(StorageEmulatorAccountName, StorageEmulatorAccountKey, DefaultBaseURL, DefaultAPIVersion, false)
}
// NewClient constructs a Client. This should be used if the caller wants
// to specify whether to use HTTPS, a specific REST API version or a custom
// storage endpoint than Azure Public Cloud.
func NewClient(accountName, accountKey, blobServiceBaseURL, apiVersion string, useHTTPS bool) (Client, error) {
var c Client
if accountName == "" {
return c, fmt.Errorf("azure: account name required")
} else if accountKey == "" {
return c, fmt.Errorf("azure: account key required")
} else if blobServiceBaseURL == "" {
return c, fmt.Errorf("azure: base storage service url required")
}
key, err := base64.StdEncoding.DecodeString(accountKey)
if err != nil {
return c, fmt.Errorf("azure: malformed storage account key: %v", err)
}
return Client{
accountName: accountName,
accountKey: key,
useHTTPS: useHTTPS,
baseURL: blobServiceBaseURL,
apiVersion: apiVersion,
}, nil
}
func (c Client) getBaseURL(service string) string {
scheme := "http"
if c.useHTTPS {
scheme = "https"
}
host := ""
if c.accountName == StorageEmulatorAccountName {
switch service {
case blobServiceName:
host = storageEmulatorBlob
case tableServiceName:
host = storageEmulatorTable
case queueServiceName:
host = storageEmulatorQueue
}
} else {
host = fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseURL)
}
u := &url.URL{
Scheme: scheme,
Host: host}
return u.String()
}
func (c Client) getEndpoint(service, path string, params url.Values) string {
u, err := url.Parse(c.getBaseURL(service))
if err != nil {
// really should not be happening
panic(err)
}
// API doesn't accept path segments not starting with '/'
if !strings.HasPrefix(path, "/") {
path = fmt.Sprintf("/%v", path)
}
if c.accountName == StorageEmulatorAccountName {
path = fmt.Sprintf("/%v%v", StorageEmulatorAccountName, path)
}
u.Path = path
u.RawQuery = params.Encode()
return u.String()
}
// GetBlobService returns a BlobStorageClient which can operate on the blob
// service of the storage account.
func (c Client) GetBlobService() BlobStorageClient {
return BlobStorageClient{c}
}
// GetQueueService returns a QueueServiceClient which can operate on the queue
// service of the storage account.
func (c Client) GetQueueService() QueueServiceClient {
return QueueServiceClient{c}
}
// GetTableService returns a TableServiceClient which can operate on the table
// service of the storage account.
func (c Client) GetTableService() TableServiceClient {
return TableServiceClient{c}
}
// GetFileService returns a FileServiceClient which can operate on the file
// service of the storage account.
func (c Client) GetFileService() FileServiceClient {
return FileServiceClient{c}
}
func (c Client) createAuthorizationHeader(canonicalizedString string) string {
signature := c.computeHmac256(canonicalizedString)
return fmt.Sprintf("%s %s:%s", "SharedKey", c.getCanonicalizedAccountName(), signature)
}
func (c Client) getAuthorizationHeader(verb, url string, headers map[string]string) (string, error) {
canonicalizedResource, err := c.buildCanonicalizedResource(url)
if err != nil {
return "", err
}
canonicalizedString := c.buildCanonicalizedString(verb, headers, canonicalizedResource)
return c.createAuthorizationHeader(canonicalizedString), nil
}
func (c Client) getStandardHeaders() map[string]string {
return map[string]string{
"x-ms-version": c.apiVersion,
"x-ms-date": currentTimeRfc1123Formatted(),
}
}
func (c Client) getCanonicalizedAccountName() string {
// since we may be trying to access a secondary storage account, we need to
// remove the -secondary part of the storage name
return strings.TrimSuffix(c.accountName, "-secondary")
}
func (c Client) buildCanonicalizedHeader(headers map[string]string) string {
cm := make(map[string]string)
for k, v := range headers {
headerName := strings.TrimSpace(strings.ToLower(k))
match, _ := regexp.MatchString("x-ms-", headerName)
if match {
cm[headerName] = v
}
}
if len(cm) == 0 {
return ""
}
keys := make([]string, 0, len(cm))
for key := range cm {
keys = append(keys, key)
}
sort.Strings(keys)
ch := ""
for i, key := range keys {
if i == len(keys)-1 {
ch += fmt.Sprintf("%s:%s", key, cm[key])
} else {
ch += fmt.Sprintf("%s:%s\n", key, cm[key])
}
}
return ch
}
func (c Client) buildCanonicalizedResourceTable(uri string) (string, error) {
errMsg := "buildCanonicalizedResourceTable error: %s"
u, err := url.Parse(uri)
if err != nil {
return "", fmt.Errorf(errMsg, err.Error())
}
cr := "/" + c.getCanonicalizedAccountName()
if len(u.Path) > 0 {
cr += u.Path
}
return cr, nil
}
func (c Client) buildCanonicalizedResource(uri string) (string, error) {
errMsg := "buildCanonicalizedResource error: %s"
u, err := url.Parse(uri)
if err != nil {
return "", fmt.Errorf(errMsg, err.Error())
}
cr := "/" + c.getCanonicalizedAccountName()
if len(u.Path) > 0 {
// Any portion of the CanonicalizedResource string that is derived from
// the resource's URI should be encoded exactly as it is in the URI.
// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
cr += u.EscapedPath()
}
params, err := url.ParseQuery(u.RawQuery)
if err != nil {
return "", fmt.Errorf(errMsg, err.Error())
}
if len(params) > 0 {
cr += "\n"
keys := make([]string, 0, len(params))
for key := range params {
keys = append(keys, key)
}
sort.Strings(keys)
for i, key := range keys {
if len(params[key]) > 1 {
sort.Strings(params[key])
}
if i == len(keys)-1 {
cr += fmt.Sprintf("%s:%s", key, strings.Join(params[key], ","))
} else {
cr += fmt.Sprintf("%s:%s\n", key, strings.Join(params[key], ","))
}
}
}
return cr, nil
}
func (c Client) buildCanonicalizedString(verb string, headers map[string]string, canonicalizedResource string) string {
contentLength := headers["Content-Length"]
if contentLength == "0" {
contentLength = ""
}
canonicalizedString := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s",
verb,
headers["Content-Encoding"],
headers["Content-Language"],
contentLength,
headers["Content-MD5"],
headers["Content-Type"],
headers["Date"],
headers["If-Modified-Since"],
headers["If-Match"],
headers["If-None-Match"],
headers["If-Unmodified-Since"],
headers["Range"],
c.buildCanonicalizedHeader(headers),
canonicalizedResource)
return canonicalizedString
}
func (c Client) exec(verb, url string, headers map[string]string, body io.Reader) (*storageResponse, error) {
authHeader, err := c.getAuthorizationHeader(verb, url, headers)
if err != nil {
return nil, err
}
headers["Authorization"] = authHeader
if err != nil {
return nil, err
}
req, err := http.NewRequest(verb, url, body)
if err != nil {
return nil, errors.New("azure/storage: error creating request: " + err.Error())
}
if clstr, ok := headers["Content-Length"]; ok {
// content length header is being signed, but completely ignored by golang.
// instead we have to use the ContentLength property on the request struct
// (see https://golang.org/src/net/http/request.go?s=18140:18370#L536 and
// https://golang.org/src/net/http/transfer.go?s=1739:2467#L49)
req.ContentLength, err = strconv.ParseInt(clstr, 10, 64)
if err != nil {
return nil, err
}
}
for k, v := range headers {
req.Header.Add(k, v)
}
httpClient := c.HTTPClient
if httpClient == nil {
httpClient = http.DefaultClient
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
statusCode := resp.StatusCode
if statusCode >= 400 && statusCode <= 505 {
var respBody []byte
respBody, err = readResponseBody(resp)
if err != nil {
return nil, err
}
if len(respBody) == 0 {
// no error in response body
err = fmt.Errorf("storage: service returned without a response body (%s)", resp.Status)
} else {
// response contains storage service error object, unmarshal
storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, resp.Header.Get("x-ms-request-id"))
if err != nil { // error unmarshaling the error response
err = errIn
}
err = storageErr
}
return &storageResponse{
statusCode: resp.StatusCode,
headers: resp.Header,
body: ioutil.NopCloser(bytes.NewReader(respBody)), /* restore the body */
}, err
}
return &storageResponse{
statusCode: resp.StatusCode,
headers: resp.Header,
body: resp.Body}, nil
}
func (c Client) execInternalJSON(verb, url string, headers map[string]string, body io.Reader) (*odataResponse, error) {
req, err := http.NewRequest(verb, url, body)
for k, v := range headers {
req.Header.Add(k, v)
}
httpClient := c.HTTPClient
if httpClient == nil {
httpClient = http.DefaultClient
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
respToRet := &odataResponse{}
respToRet.body = resp.Body
respToRet.statusCode = resp.StatusCode
respToRet.headers = resp.Header
statusCode := resp.StatusCode
if statusCode >= 400 && statusCode <= 505 {
var respBody []byte
respBody, err = readResponseBody(resp)
if err != nil {
return nil, err
}
if len(respBody) == 0 {
// no error in response body
err = fmt.Errorf("storage: service returned without a response body (%d)", resp.StatusCode)
return respToRet, err
}
// try unmarshal as odata.error json
err = json.Unmarshal(respBody, &respToRet.odata)
return respToRet, err
}
return respToRet, nil
}
func (c Client) createSharedKeyLite(url string, headers map[string]string) (string, error) {
can, err := c.buildCanonicalizedResourceTable(url)
if err != nil {
return "", err
}
strToSign := headers["x-ms-date"] + "\n" + can
hmac := c.computeHmac256(strToSign)
return fmt.Sprintf("SharedKeyLite %s:%s", c.accountName, hmac), nil
}
func (c Client) execTable(verb, url string, headers map[string]string, body io.Reader) (*odataResponse, error) {
var err error
headers["Authorization"], err = c.createSharedKeyLite(url, headers)
if err != nil {
return nil, err
}
return c.execInternalJSON(verb, url, headers, body)
}
func readResponseBody(resp *http.Response) ([]byte, error) {
defer resp.Body.Close()
out, err := ioutil.ReadAll(resp.Body)
if err == io.EOF {
err = nil
}
return out, err
}
func serviceErrFromXML(body []byte, statusCode int, requestID string) (AzureStorageServiceError, error) {
var storageErr AzureStorageServiceError
if err := xml.Unmarshal(body, &storageErr); err != nil {
return storageErr, err
}
storageErr.StatusCode = statusCode
storageErr.RequestID = requestID
return storageErr, nil
}
func (e AzureStorageServiceError) Error() string {
return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s, QueryParameterName=%s, QueryParameterValue=%s",
e.StatusCode, e.Code, e.Message, e.RequestID, e.QueryParameterName, e.QueryParameterValue)
}
// checkRespCode returns UnexpectedStatusError if the given response code is not
// one of the allowed status codes; otherwise nil.
func checkRespCode(respCode int, allowed []int) error {
for _, v := range allowed {
if respCode == v {
return nil
}
}
return UnexpectedStatusCodeError{allowed, respCode}
}