From 0a1ce58e2cda82fc61f681c1ee778c0036625a9b Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Mon, 12 Dec 2016 21:12:39 -0800 Subject: [PATCH 1/3] azure: revendor + remove hacky solution in is404 Removing the temporary workaround in is404() method by re-vendoring the azure-sdk-for-go. Signed-off-by: Ahmet Alp Balkan --- registry/storage/driver/azure/azure.go | 8 - .../Azure/azure-sdk-for-go/storage/blob.go | 348 ++++++++- .../Azure/azure-sdk-for-go/storage/client.go | 23 +- .../Azure/azure-sdk-for-go/storage/file.go | 708 +++++++++++++++--- .../Azure/azure-sdk-for-go/storage/queue.go | 38 + .../storage/table_entities.go | 2 + .../Azure/azure-sdk-for-go/storage/util.go | 2 +- 7 files changed, 1007 insertions(+), 122 deletions(-) diff --git a/registry/storage/driver/azure/azure.go b/registry/storage/driver/azure/azure.go index bec2cf8e5..cfffad1c1 100644 --- a/registry/storage/driver/azure/azure.go +++ b/registry/storage/driver/azure/azure.go @@ -397,14 +397,6 @@ func (d *driver) listBlobs(container, virtPath string) ([]string, error) { } func is404(err error) bool { - // handle the case when the request was a HEAD and service error could not - // be parsed, such as "storage: service returned without a response body - // (404 The specified blob does not exist.)" - if strings.Contains(fmt.Sprintf("%v", err), "404 The specified blob does not exist") { - return true - } - - // common case statusCodeErr, ok := err.(azure.AzureStorageServiceError) return ok && statusCodeErr.StatusCode == http.StatusNotFound } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go index f3c9ba6c8..3dbaca52a 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "net/http" "net/url" "strconv" @@ -301,6 +302,65 @@ const ( ContainerAccessTypeContainer ContainerAccessType = "container" ) +// ContainerAccessOptions are used when setting ACLs of containers (after creation) +type ContainerAccessOptions struct { + ContainerAccess ContainerAccessType + Timeout int + LeaseID string +} + +// AccessPolicyDetails are used for SETTING policies +type AccessPolicyDetails struct { + ID string + StartTime time.Time + ExpiryTime time.Time + CanRead bool + CanWrite bool + CanDelete bool +} + +// ContainerPermissions is used when setting permissions and Access Policies for containers. +type ContainerPermissions struct { + AccessOptions ContainerAccessOptions + AccessPolicy AccessPolicyDetails +} + +// AccessPolicyDetailsXML has specifics about an access policy +// annotated with XML details. +type AccessPolicyDetailsXML struct { + StartTime time.Time `xml:"Start"` + ExpiryTime time.Time `xml:"Expiry"` + Permission string `xml:"Permission"` +} + +// SignedIdentifier is a wrapper for a specific policy +type SignedIdentifier struct { + ID string `xml:"Id"` + AccessPolicy AccessPolicyDetailsXML `xml:"AccessPolicy"` +} + +// SignedIdentifiers part of the response from GetPermissions call. +type SignedIdentifiers struct { + SignedIdentifiers []SignedIdentifier `xml:"SignedIdentifier"` +} + +// AccessPolicy is the response type from the GetPermissions call. +type AccessPolicy struct { + SignedIdentifiersList SignedIdentifiers `xml:"SignedIdentifiers"` +} + +// ContainerAccessResponse is returned for the GetContainerPermissions function. +// This contains both the permission and access policy for the container. +type ContainerAccessResponse struct { + ContainerAccess ContainerAccessType + AccessPolicy SignedIdentifiers +} + +// ContainerAccessHeader references header used when setting/getting container ACL +const ( + ContainerAccessHeader string = "x-ms-blob-public-access" +) + // Maximum sizes (per REST API) for various concepts const ( MaxBlobBlockSize = 4 * 1024 * 1024 @@ -416,7 +476,7 @@ func (b BlobStorageClient) createContainer(name string, access ContainerAccessTy headers := b.client.getStandardHeaders() if access != "" { - headers["x-ms-blob-public-access"] = string(access) + headers[ContainerAccessHeader] = string(access) } return b.client.exec(verb, uri, headers, nil) } @@ -438,6 +498,101 @@ func (b BlobStorageClient) ContainerExists(name string) (bool, error) { return false, err } +// SetContainerPermissions sets up container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179391.aspx +func (b BlobStorageClient) SetContainerPermissions(container string, containerPermissions ContainerPermissions) (err error) { + params := url.Values{ + "restype": {"container"}, + "comp": {"acl"}, + } + + if containerPermissions.AccessOptions.Timeout > 0 { + params.Add("timeout", strconv.Itoa(containerPermissions.AccessOptions.Timeout)) + } + + uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params) + headers := b.client.getStandardHeaders() + if containerPermissions.AccessOptions.ContainerAccess != "" { + headers[ContainerAccessHeader] = string(containerPermissions.AccessOptions.ContainerAccess) + } + + if containerPermissions.AccessOptions.LeaseID != "" { + headers[leaseID] = containerPermissions.AccessOptions.LeaseID + } + + // generate the XML for the SharedAccessSignature if required. + accessPolicyXML, err := generateAccessPolicy(containerPermissions.AccessPolicy) + if err != nil { + return err + } + + var resp *storageResponse + if accessPolicyXML != "" { + headers["Content-Length"] = strconv.Itoa(len(accessPolicyXML)) + resp, err = b.client.exec("PUT", uri, headers, strings.NewReader(accessPolicyXML)) + } else { + resp, err = b.client.exec("PUT", uri, headers, nil) + } + + if err != nil { + return err + } + + if resp != nil { + defer func() { + err = resp.body.Close() + }() + + if resp.statusCode != http.StatusOK { + return errors.New("Unable to set permissions") + } + } + return nil +} + +// GetContainerPermissions gets the container permissions as per https://msdn.microsoft.com/en-us/library/azure/dd179469.aspx +// If timeout is 0 then it will not be passed to Azure +// leaseID will only be passed to Azure if populated +// Returns permissionResponse which is combined permissions and AccessPolicy +func (b BlobStorageClient) GetContainerPermissions(container string, timeout int, leaseID string) (permissionResponse *ContainerAccessResponse, err error) { + params := url.Values{"restype": {"container"}, + "comp": {"acl"}} + + if timeout > 0 { + params.Add("timeout", strconv.Itoa(timeout)) + } + + uri := b.client.getEndpoint(blobServiceName, pathForContainer(container), params) + headers := b.client.getStandardHeaders() + + if leaseID != "" { + headers[leaseID] = leaseID + } + + resp, err := b.client.exec("GET", uri, headers, nil) + if err != nil { + return nil, err + } + + // containerAccess. Blob, Container, empty + containerAccess := resp.headers.Get(http.CanonicalHeaderKey(ContainerAccessHeader)) + + defer func() { + err = resp.body.Close() + }() + + var out AccessPolicy + err = xmlUnmarshal(resp.body, &out.SignedIdentifiersList) + if err != nil { + return nil, err + } + + permissionResponse = &ContainerAccessResponse{} + permissionResponse.AccessPolicy = out.SignedIdentifiersList + permissionResponse.ContainerAccess = ContainerAccessType(containerAccess) + + return permissionResponse, nil +} + // DeleteContainer deletes the container with given name on the storage // account. If the container does not exist returns error. // @@ -595,13 +750,55 @@ func (b BlobStorageClient) leaseCommonPut(container string, name string, headers return resp.headers, nil } +// SnapshotBlob creates a snapshot for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691971.aspx +func (b BlobStorageClient) SnapshotBlob(container string, name string, timeout int, extraHeaders map[string]string) (snapshotTimestamp *time.Time, err error) { + headers := b.client.getStandardHeaders() + params := url.Values{"comp": {"snapshot"}} + + if timeout > 0 { + params.Add("timeout", strconv.Itoa(timeout)) + } + + for k, v := range extraHeaders { + headers[k] = v + } + + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) + resp, err := b.client.exec("PUT", uri, headers, nil) + if err != nil { + return nil, err + } + + if err := checkRespCode(resp.statusCode, []int{http.StatusCreated}); err != nil { + return nil, err + } + + snapshotResponse := resp.headers.Get(http.CanonicalHeaderKey("x-ms-snapshot")) + if snapshotResponse != "" { + snapshotTimestamp, err := time.Parse(time.RFC3339, snapshotResponse) + if err != nil { + return nil, err + } + + return &snapshotTimestamp, nil + } + + return nil, errors.New("Snapshot not created") +} + // AcquireLease creates a lease for a blob as per https://msdn.microsoft.com/en-us/library/azure/ee691972.aspx // returns leaseID acquired func (b BlobStorageClient) AcquireLease(container string, name string, leaseTimeInSeconds int, proposedLeaseID string) (returnedLeaseID string, err error) { headers := b.client.getStandardHeaders() headers[leaseAction] = acquireLease - headers[leaseProposedID] = proposedLeaseID - headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds) + + if leaseTimeInSeconds > 0 { + headers[leaseDuration] = strconv.Itoa(leaseTimeInSeconds) + } + + if proposedLeaseID != "" { + headers[leaseProposedID] = proposedLeaseID + } respHeaders, err := b.leaseCommonPut(container, name, headers, http.StatusCreated) if err != nil { @@ -614,8 +811,6 @@ func (b BlobStorageClient) AcquireLease(container string, name string, leaseTime return returnedLeaseID, nil } - // what should we return in case of HTTP 201 but no lease ID? - // or it just cant happen? (brave words) return "", errors.New("LeaseID not returned") } @@ -1106,15 +1301,20 @@ func (b BlobStorageClient) AppendBlock(container, name string, chunk []byte, ext // // See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx func (b BlobStorageClient) CopyBlob(container, name, sourceBlob string) error { - copyID, err := b.startBlobCopy(container, name, sourceBlob) + copyID, err := b.StartBlobCopy(container, name, sourceBlob) if err != nil { return err } - return b.waitForBlobCopy(container, name, copyID) + return b.WaitForBlobCopy(container, name, copyID) } -func (b BlobStorageClient) startBlobCopy(container, name, sourceBlob string) (string, error) { +// StartBlobCopy starts a blob copy operation. +// sourceBlob parameter must be a canonical URL to the blob (can be +// obtained using GetBlobURL method.) +// +// See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx +func (b BlobStorageClient) StartBlobCopy(container, name, sourceBlob string) (string, error) { uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), url.Values{}) headers := b.client.getStandardHeaders() @@ -1137,7 +1337,39 @@ func (b BlobStorageClient) startBlobCopy(container, name, sourceBlob string) (st return copyID, nil } -func (b BlobStorageClient) waitForBlobCopy(container, name, copyID string) error { +// AbortBlobCopy aborts a BlobCopy which has already been triggered by the StartBlobCopy function. +// copyID is generated from StartBlobCopy function. +// currentLeaseID is required IF the destination blob has an active lease on it. +// As defined in https://msdn.microsoft.com/en-us/library/azure/jj159098.aspx +func (b BlobStorageClient) AbortBlobCopy(container, name, copyID, currentLeaseID string, timeout int) error { + params := url.Values{"comp": {"copy"}, "copyid": {copyID}} + if timeout > 0 { + params.Add("timeout", strconv.Itoa(timeout)) + } + + uri := b.client.getEndpoint(blobServiceName, pathForBlob(container, name), params) + headers := b.client.getStandardHeaders() + headers["x-ms-copy-action"] = "abort" + + if currentLeaseID != "" { + headers[leaseID] = currentLeaseID + } + + resp, err := b.client.exec("PUT", uri, headers, nil) + if err != nil { + return err + } + defer resp.body.Close() + + if err := checkRespCode(resp.statusCode, []int{http.StatusNoContent}); err != nil { + return err + } + + return nil +} + +// WaitForBlobCopy loops until a BlobCopy operation is completed (or fails with error) +func (b BlobStorageClient) WaitForBlobCopy(container, name, copyID string) error { for { props, err := b.GetBlobProperties(container, name) if err != nil { @@ -1212,17 +1444,18 @@ func pathForBlob(container, name string) string { return fmt.Sprintf("/%s/%s", container, name) } -// GetBlobSASURI creates an URL to the specified blob which contains the Shared -// Access Signature with specified permissions and expiration time. +// GetBlobSASURIWithSignedIPAndProtocol creates an URL to the specified blob which contains the Shared +// Access Signature with specified permissions and expiration time. Also includes signedIPRange and allowed procotols. +// If old API version is used but no signedIP is passed (ie empty string) then this should still work. +// We only populate the signedIP when it non-empty. // // See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx -func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Time, permissions string) (string, error) { +func (b BlobStorageClient) GetBlobSASURIWithSignedIPAndProtocol(container, name string, expiry time.Time, permissions string, signedIPRange string, HTTPSOnly bool) (string, error) { var ( signedPermissions = permissions blobURL = b.GetBlobURL(container, name) ) canonicalizedResource, err := b.client.buildCanonicalizedResource(blobURL) - if err != nil { return "", err } @@ -1234,7 +1467,6 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim // We need to replace + with %2b first to avoid being treated as a space (which is correct for query strings, but not the path component). canonicalizedResource = strings.Replace(canonicalizedResource, "+", "%2b", -1) - canonicalizedResource, err = url.QueryUnescape(canonicalizedResource) if err != nil { return "", err @@ -1243,7 +1475,11 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim signedExpiry := expiry.UTC().Format(time.RFC3339) signedResource := "b" - stringToSign, err := blobSASStringToSign(b.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions) + protocols := "https,http" + if HTTPSOnly { + protocols = "https" + } + stringToSign, err := blobSASStringToSign(b.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions, signedIPRange, protocols) if err != nil { return "", err } @@ -1257,6 +1493,13 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim "sig": {sig}, } + if b.client.apiVersion >= "2015-04-05" { + sasParams.Add("spr", protocols) + if signedIPRange != "" { + sasParams.Add("sip", signedIPRange) + } + } + sasURL, err := url.Parse(blobURL) if err != nil { return "", err @@ -1265,16 +1508,89 @@ func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Tim return sasURL.String(), nil } -func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string) (string, error) { +// GetBlobSASURI creates an URL to the specified blob which contains the Shared +// Access Signature with specified permissions and expiration time. +// +// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx +func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Time, permissions string) (string, error) { + url, err := b.GetBlobSASURIWithSignedIPAndProtocol(container, name, expiry, permissions, "", false) + return url, err +} + +func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string, signedIP string, protocols string) (string, error) { var signedStart, signedIdentifier, rscc, rscd, rsce, rscl, rsct string if signedVersion >= "2015-02-21" { canonicalizedResource = "/blob" + canonicalizedResource } + // https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx#Anchor_12 + if signedVersion >= "2015-04-05" { + return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedIP, protocols, signedVersion, rscc, rscd, rsce, rscl, rsct), nil + } + // reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx if signedVersion >= "2013-08-15" { return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion, rscc, rscd, rsce, rscl, rsct), nil } + return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15") } + +func generatePermissions(accessPolicy AccessPolicyDetails) (permissions string) { + // generate the permissions string (rwd). + // still want the end user API to have bool flags. + permissions = "" + + if accessPolicy.CanRead { + permissions += "r" + } + + if accessPolicy.CanWrite { + permissions += "w" + } + + if accessPolicy.CanDelete { + permissions += "d" + } + + return permissions +} + +// convertAccessPolicyToXMLStructs converts between AccessPolicyDetails which is a struct better for API usage to the +// AccessPolicy struct which will get converted to XML. +func convertAccessPolicyToXMLStructs(accessPolicy AccessPolicyDetails) SignedIdentifiers { + return SignedIdentifiers{ + SignedIdentifiers: []SignedIdentifier{ + { + ID: accessPolicy.ID, + AccessPolicy: AccessPolicyDetailsXML{ + StartTime: accessPolicy.StartTime.UTC().Round(time.Second), + ExpiryTime: accessPolicy.ExpiryTime.UTC().Round(time.Second), + Permission: generatePermissions(accessPolicy), + }, + }, + }, + } +} + +// generateAccessPolicy generates the XML access policy used as the payload for SetContainerPermissions. +func generateAccessPolicy(accessPolicy AccessPolicyDetails) (accessPolicyXML string, err error) { + + if accessPolicy.ID != "" { + signedIdentifiers := convertAccessPolicyToXMLStructs(accessPolicy) + body, _, err := xmlMarshal(signedIdentifiers) + if err != nil { + return "", err + } + + xmlByteArray, err := ioutil.ReadAll(body) + if err != nil { + return "", err + } + accessPolicyXML = string(xmlByteArray) + return accessPolicyXML, nil + } + + return "", nil +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go index 2816e03ec..ad5affa43 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go @@ -128,6 +128,7 @@ func NewBasicClient(accountName, accountKey string) (Client, error) { return NewEmulatorClient() } return NewClient(accountName, accountKey, DefaultBaseURL, DefaultAPIVersion, defaultUseHTTPS) + } //NewEmulatorClient contructs a Client intended to only work with Azure @@ -305,7 +306,7 @@ func (c Client) buildCanonicalizedResourceTable(uri string) (string, error) { cr := "/" + c.getCanonicalizedAccountName() if len(u.Path) > 0 { - cr += u.Path + cr += u.EscapedPath() } return cr, nil @@ -427,12 +428,13 @@ func (c Client) exec(verb, url string, headers map[string]string, body io.Reader return nil, err } + requestID := resp.Header.Get("x-ms-request-id") if len(respBody) == 0 { - // no error in response body - err = fmt.Errorf("storage: service returned without a response body (%s)", resp.Status) + // no error in response body, might happen in HEAD requests + err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID) } else { // response contains storage service error object, unmarshal - storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, resp.Header.Get("x-ms-request-id")) + storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, requestID) if err != nil { // error unmarshaling the error response err = errIn } @@ -481,8 +483,8 @@ func (c Client) execInternalJSON(verb, url string, headers map[string]string, bo } if len(respBody) == 0 { - // no error in response body - err = fmt.Errorf("storage: service returned without a response body (%d)", resp.StatusCode) + // no error in response body, might happen in HEAD requests + err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, resp.Header.Get("x-ms-request-id")) return respToRet, err } // try unmarshal as odata.error json @@ -534,6 +536,15 @@ func serviceErrFromXML(body []byte, statusCode int, requestID string) (AzureStor return storageErr, nil } +func serviceErrFromStatusCode(code int, status string, requestID string) AzureStorageServiceError { + return AzureStorageServiceError{ + StatusCode: code, + Code: status, + RequestID: requestID, + Message: "no response body was available for error status code", + } +} + func (e AzureStorageServiceError) Error() string { return fmt.Sprintf("storage: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s, QueryParameterName=%s, QueryParameterValue=%s", e.StatusCode, e.Code, e.Message, e.RequestID, e.QueryParameterName, e.QueryParameterValue) diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go index 2397587c8..f679395bd 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go @@ -2,9 +2,12 @@ package storage import ( "encoding/xml" + "errors" "fmt" + "io" "net/http" "net/url" + "strconv" "strings" ) @@ -19,6 +22,17 @@ type Share struct { Properties ShareProperties `xml:"Properties"` } +// A Directory is an entry in DirsAndFilesListResponse. +type Directory struct { + Name string `xml:"Name"` +} + +// A File is an entry in DirsAndFilesListResponse. +type File struct { + Name string `xml:"Name"` + Properties FileProperties `xml:"Properties"` +} + // ShareProperties contains various properties of a share returned from // various endpoints like ListShares. type ShareProperties struct { @@ -27,6 +41,40 @@ type ShareProperties struct { Quota string `xml:"Quota"` } +// DirectoryProperties contains various properties of a directory returned +// from various endpoints like GetDirectoryProperties. +type DirectoryProperties struct { + LastModified string `xml:"Last-Modified"` + Etag string `xml:"Etag"` +} + +// FileProperties contains various properties of a file returned from +// various endpoints like ListDirsAndFiles. +type FileProperties struct { + CacheControl string `header:"x-ms-cache-control"` + ContentLength uint64 `xml:"Content-Length"` + ContentType string `header:"x-ms-content-type"` + CopyCompletionTime string + CopyID string + CopySource string + CopyProgress string + CopyStatusDesc string + CopyStatus string + Disposition string `header:"x-ms-content-disposition"` + Encoding string `header:"x-ms-content-encoding"` + Etag string + Language string `header:"x-ms-content-language"` + LastModified string + MD5 string `header:"x-ms-content-md5"` +} + +// FileStream contains file data returned from a call to GetFile. +type FileStream struct { + Body io.ReadCloser + Properties *FileProperties + Metadata map[string]string +} + // ShareListResponse contains the response fields from // ListShares call. // @@ -53,12 +101,80 @@ type ListSharesParameters struct { Timeout uint } +// DirsAndFilesListResponse contains the response fields from +// a List Files and Directories call. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166980.aspx +type DirsAndFilesListResponse struct { + XMLName xml.Name `xml:"EnumerationResults"` + Xmlns string `xml:"xmlns,attr"` + Marker string `xml:"Marker"` + MaxResults int64 `xml:"MaxResults"` + Directories []Directory `xml:"Entries>Directory"` + Files []File `xml:"Entries>File"` + NextMarker string `xml:"NextMarker"` +} + +// FileRanges contains a list of file range information for a file. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166984.aspx +type FileRanges struct { + ContentLength uint64 + LastModified string + ETag string + FileRanges []FileRange `xml:"Range"` +} + +// FileRange contains range information for a file. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166984.aspx +type FileRange struct { + Start uint64 `xml:"Start"` + End uint64 `xml:"End"` +} + +// ListDirsAndFilesParameters defines the set of customizable parameters to +// make a List Files and Directories call. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166980.aspx +type ListDirsAndFilesParameters struct { + Marker string + MaxResults uint + Timeout uint +} + // ShareHeaders contains various properties of a file and is an entry // in SetShareProperties type ShareHeaders struct { Quota string `header:"x-ms-share-quota"` } +type compType string + +const ( + compNone compType = "" + compList compType = "list" + compMetadata compType = "metadata" + compProperties compType = "properties" + compRangeList compType = "rangelist" +) + +func (ct compType) String() string { + return string(ct) +} + +type resourceType string + +const ( + resourceDirectory resourceType = "directory" + resourceFile resourceType = "" + resourceShare resourceType = "share" +) + +func (rt resourceType) String() string { + return string(rt) +} + func (p ListSharesParameters) getParameters() url.Values { out := url.Values{} @@ -81,9 +197,97 @@ func (p ListSharesParameters) getParameters() url.Values { return out } -// pathForFileShare returns the URL path segment for a File Share resource -func pathForFileShare(name string) string { - return fmt.Sprintf("/%s", name) +func (p ListDirsAndFilesParameters) getParameters() url.Values { + out := url.Values{} + + if p.Marker != "" { + out.Set("marker", p.Marker) + } + if p.MaxResults != 0 { + out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults)) + } + if p.Timeout != 0 { + out.Set("timeout", fmt.Sprintf("%v", p.Timeout)) + } + + return out +} + +func (fr FileRange) String() string { + return fmt.Sprintf("bytes=%d-%d", fr.Start, fr.End) +} + +// ToPathSegment returns the URL path segment for the specified values +func ToPathSegment(parts ...string) string { + join := strings.Join(parts, "/") + if join[0] != '/' { + join = fmt.Sprintf("/%s", join) + } + return join +} + +// returns url.Values for the specified types +func getURLInitValues(comp compType, res resourceType) url.Values { + values := url.Values{} + if comp != compNone { + values.Set("comp", comp.String()) + } + if res != resourceFile { + values.Set("restype", res.String()) + } + return values +} + +// ListDirsAndFiles returns a list of files or directories under the specified share or +// directory. It also contains a pagination token and other response details. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166980.aspx +func (f FileServiceClient) ListDirsAndFiles(path string, params ListDirsAndFilesParameters) (DirsAndFilesListResponse, error) { + q := mergeParams(params.getParameters(), getURLInitValues(compList, resourceDirectory)) + + var out DirsAndFilesListResponse + resp, err := f.listContent(path, q, nil) + if err != nil { + return out, err + } + + defer resp.body.Close() + err = xmlUnmarshal(resp.body, &out) + return out, err +} + +// ListFileRanges returns the list of valid ranges for a file. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166984.aspx +func (f FileServiceClient) ListFileRanges(path string, listRange *FileRange) (FileRanges, error) { + params := url.Values{"comp": {"rangelist"}} + + // add optional range to list + var headers map[string]string + if listRange != nil { + headers = make(map[string]string) + headers["Range"] = listRange.String() + } + + var out FileRanges + resp, err := f.listContent(path, params, headers) + if err != nil { + return out, err + } + + defer resp.body.Close() + var cl uint64 + cl, err = strconv.ParseUint(resp.headers.Get("x-ms-content-length"), 10, 64) + if err != nil { + return out, err + } + + out.ContentLength = cl + out.ETag = resp.headers.Get("ETag") + out.LastModified = resp.headers.Get("Last-Modified") + + err = xmlUnmarshal(resp.body, &out) + return out, err } // ListShares returns the list of shares in a storage account along with @@ -92,40 +296,176 @@ func pathForFileShare(name string) string { // See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx func (f FileServiceClient) ListShares(params ListSharesParameters) (ShareListResponse, error) { q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}}) - uri := f.client.getEndpoint(fileServiceName, "", q) - headers := f.client.getStandardHeaders() var out ShareListResponse - resp, err := f.client.exec("GET", uri, headers, nil) + resp, err := f.listContent("", q, nil) if err != nil { return out, err } - defer resp.body.Close() + defer resp.body.Close() err = xmlUnmarshal(resp.body, &out) return out, err } -// CreateShare operation creates a new share under the specified account. If the -// share with the same name already exists, the operation fails. +// retrieves directory or share content +func (f FileServiceClient) listContent(path string, params url.Values, extraHeaders map[string]string) (*storageResponse, error) { + if err := f.checkForStorageEmulator(); err != nil { + return nil, err + } + + uri := f.client.getEndpoint(fileServiceName, path, params) + headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) + + resp, err := f.client.exec(http.MethodGet, uri, headers, nil) + if err != nil { + return nil, err + } + + if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + resp.body.Close() + return nil, err + } + + return resp, nil +} + +// CreateDirectory operation creates a new directory with optional metadata in the +// specified share. If a directory with the same name already exists, the operation fails. // -// See https://msdn.microsoft.com/en-us/library/azure/dn167008.aspx -func (f FileServiceClient) CreateShare(name string) error { - resp, err := f.createShare(name) +// See https://msdn.microsoft.com/en-us/library/azure/dn166993.aspx +func (f FileServiceClient) CreateDirectory(path string, metadata map[string]string) error { + return f.createResource(path, resourceDirectory, mergeMDIntoExtraHeaders(metadata, nil)) +} + +// CreateFile operation creates a new file with optional metadata or replaces an existing one. +// Note that this only initializes the file, call PutRange to add content. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn194271.aspx +func (f FileServiceClient) CreateFile(path string, maxSize uint64, metadata map[string]string) error { + extraHeaders := map[string]string{ + "x-ms-content-length": strconv.FormatUint(maxSize, 10), + "x-ms-type": "file", + } + return f.createResource(path, resourceFile, mergeMDIntoExtraHeaders(metadata, extraHeaders)) +} + +// ClearRange releases the specified range of space in storage. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn194276.aspx +func (f FileServiceClient) ClearRange(path string, fileRange FileRange) error { + return f.modifyRange(path, nil, fileRange) +} + +// PutRange writes a range of bytes to a file. Note that the length of bytes must +// match (rangeEnd - rangeStart) + 1 with a maximum size of 4MB. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn194276.aspx +func (f FileServiceClient) PutRange(path string, bytes io.Reader, fileRange FileRange) error { + return f.modifyRange(path, bytes, fileRange) +} + +// modifies a range of bytes in the specified file +func (f FileServiceClient) modifyRange(path string, bytes io.Reader, fileRange FileRange) error { + if err := f.checkForStorageEmulator(); err != nil { + return err + } + if fileRange.End < fileRange.Start { + return errors.New("the value for rangeEnd must be greater than or equal to rangeStart") + } + if bytes != nil && fileRange.End-fileRange.Start > 4194304 { + return errors.New("range cannot exceed 4MB in size") + } + + uri := f.client.getEndpoint(fileServiceName, path, url.Values{"comp": {"range"}}) + + // default to clear + write := "clear" + cl := uint64(0) + + // if bytes is not nil then this is an update operation + if bytes != nil { + write = "update" + cl = (fileRange.End - fileRange.Start) + 1 + } + + extraHeaders := map[string]string{ + "Content-Length": strconv.FormatUint(cl, 10), + "Range": fileRange.String(), + "x-ms-write": write, + } + + headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) + resp, err := f.client.exec(http.MethodPut, uri, headers, bytes) if err != nil { return err } + defer resp.body.Close() return checkRespCode(resp.statusCode, []int{http.StatusCreated}) } +// GetFile operation reads or downloads a file from the system, including its +// metadata and properties. +// +// See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/get-file +func (f FileServiceClient) GetFile(path string, fileRange *FileRange) (*FileStream, error) { + var extraHeaders map[string]string + if fileRange != nil { + extraHeaders = map[string]string{ + "Range": fileRange.String(), + } + } + + resp, err := f.getResourceNoClose(path, compNone, resourceFile, http.MethodGet, extraHeaders) + if err != nil { + return nil, err + } + + if err = checkRespCode(resp.statusCode, []int{http.StatusOK, http.StatusPartialContent}); err != nil { + resp.body.Close() + return nil, err + } + + props, err := getFileProps(resp.headers) + md := getFileMDFromHeaders(resp.headers) + return &FileStream{Body: resp.body, Properties: props, Metadata: md}, nil +} + +// CreateShare operation creates a new share with optional metadata under the specified account. +// If the share with the same name already exists, the operation fails. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn167008.aspx +func (f FileServiceClient) CreateShare(name string, metadata map[string]string) error { + return f.createResource(ToPathSegment(name), resourceShare, mergeMDIntoExtraHeaders(metadata, nil)) +} + +// DirectoryExists returns true if the specified directory exists on the specified share. +func (f FileServiceClient) DirectoryExists(path string) (bool, error) { + return f.resourceExists(path, resourceDirectory) +} + +// FileExists returns true if the specified file exists. +func (f FileServiceClient) FileExists(path string) (bool, error) { + return f.resourceExists(path, resourceFile) +} + // ShareExists returns true if a share with given name exists // on the storage account, otherwise returns false. func (f FileServiceClient) ShareExists(name string) (bool, error) { - uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), url.Values{"restype": {"share"}}) + return f.resourceExists(ToPathSegment(name), resourceShare) +} + +// returns true if the specified directory or share exists +func (f FileServiceClient) resourceExists(path string, res resourceType) (bool, error) { + if err := f.checkForStorageEmulator(); err != nil { + return false, err + } + + uri := f.client.getEndpoint(fileServiceName, path, getURLInitValues(compNone, res)) headers := f.client.getStandardHeaders() - resp, err := f.client.exec("HEAD", uri, headers, nil) + resp, err := f.client.exec(http.MethodHead, uri, headers, nil) if resp != nil { defer resp.body.Close() if resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound { @@ -135,21 +475,27 @@ func (f FileServiceClient) ShareExists(name string) (bool, error) { return false, err } -// GetShareURL gets the canonical URL to the share with the specified name in the -// specified container. This method does not create a publicly accessible URL if -// the file is private and this method does not check if the file -// exists. -func (f FileServiceClient) GetShareURL(name string) string { - return f.client.getEndpoint(fileServiceName, pathForFileShare(name), url.Values{}) +// GetDirectoryURL gets the canonical URL to the directory with the specified name +// in the specified share. This method does not create a publicly accessible URL if +// the file is private and this method does not check if the directory exists. +func (f FileServiceClient) GetDirectoryURL(path string) string { + return f.client.getEndpoint(fileServiceName, path, url.Values{}) } -// CreateShareIfNotExists creates a new share under the specified account if -// it does not exist. Returns true if container is newly created or false if -// container already exists. +// GetShareURL gets the canonical URL to the share with the specified name in the +// specified container. This method does not create a publicly accessible URL if +// the file is private and this method does not check if the share exists. +func (f FileServiceClient) GetShareURL(name string) string { + return f.client.getEndpoint(fileServiceName, ToPathSegment(name), url.Values{}) +} + +// CreateDirectoryIfNotExists creates a new directory on the specified share +// if it does not exist. Returns true if directory is newly created or false +// if the directory already exists. // -// See https://msdn.microsoft.com/en-us/library/azure/dn167008.aspx -func (f FileServiceClient) CreateShareIfNotExists(name string) (bool, error) { - resp, err := f.createShare(name) +// See https://msdn.microsoft.com/en-us/library/azure/dn166993.aspx +func (f FileServiceClient) CreateDirectoryIfNotExists(path string) (bool, error) { + resp, err := f.createResourceNoClose(path, resourceDirectory, nil) if resp != nil { defer resp.body.Close() if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict { @@ -159,37 +505,149 @@ func (f FileServiceClient) CreateShareIfNotExists(name string) (bool, error) { return false, err } -// CreateShare creates a Azure File Share and returns its response -func (f FileServiceClient) createShare(name string) (*storageResponse, error) { +// CreateShareIfNotExists creates a new share under the specified account if +// it does not exist. Returns true if container is newly created or false if +// container already exists. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn167008.aspx +func (f FileServiceClient) CreateShareIfNotExists(name string) (bool, error) { + resp, err := f.createResourceNoClose(ToPathSegment(name), resourceShare, nil) + if resp != nil { + defer resp.body.Close() + if resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict { + return resp.statusCode == http.StatusCreated, nil + } + } + return false, err +} + +// creates a resource depending on the specified resource type +func (f FileServiceClient) createResource(path string, res resourceType, extraHeaders map[string]string) error { + resp, err := f.createResourceNoClose(path, res, extraHeaders) + if err != nil { + return err + } + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusCreated}) +} + +// creates a resource depending on the specified resource type, doesn't close the response body +func (f FileServiceClient) createResourceNoClose(path string, res resourceType, extraHeaders map[string]string) (*storageResponse, error) { if err := f.checkForStorageEmulator(); err != nil { return nil, err } - uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), url.Values{"restype": {"share"}}) - headers := f.client.getStandardHeaders() - return f.client.exec("PUT", uri, headers, nil) + + values := getURLInitValues(compNone, res) + uri := f.client.getEndpoint(fileServiceName, path, values) + headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) + + return f.client.exec(http.MethodPut, uri, headers, nil) +} + +// GetDirectoryProperties provides various information about the specified directory. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn194272.aspx +func (f FileServiceClient) GetDirectoryProperties(path string) (*DirectoryProperties, error) { + headers, err := f.getResourceHeaders(path, compNone, resourceDirectory, http.MethodHead) + if err != nil { + return nil, err + } + + return &DirectoryProperties{ + LastModified: headers.Get("Last-Modified"), + Etag: headers.Get("Etag"), + }, nil +} + +// GetFileProperties provides various information about the specified file. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166971.aspx +func (f FileServiceClient) GetFileProperties(path string) (*FileProperties, error) { + headers, err := f.getResourceHeaders(path, compNone, resourceFile, http.MethodHead) + if err != nil { + return nil, err + } + return getFileProps(headers) +} + +// returns file properties from the specified HTTP header +func getFileProps(header http.Header) (*FileProperties, error) { + size, err := strconv.ParseUint(header.Get("Content-Length"), 10, 64) + if err != nil { + return nil, err + } + + return &FileProperties{ + CacheControl: header.Get("Cache-Control"), + ContentLength: size, + ContentType: header.Get("Content-Type"), + CopyCompletionTime: header.Get("x-ms-copy-completion-time"), + CopyID: header.Get("x-ms-copy-id"), + CopyProgress: header.Get("x-ms-copy-progress"), + CopySource: header.Get("x-ms-copy-source"), + CopyStatus: header.Get("x-ms-copy-status"), + CopyStatusDesc: header.Get("x-ms-copy-status-description"), + Disposition: header.Get("Content-Disposition"), + Encoding: header.Get("Content-Encoding"), + Etag: header.Get("ETag"), + Language: header.Get("Content-Language"), + LastModified: header.Get("Last-Modified"), + MD5: header.Get("Content-MD5"), + }, nil } // GetShareProperties provides various information about the specified // file. See https://msdn.microsoft.com/en-us/library/azure/dn689099.aspx func (f FileServiceClient) GetShareProperties(name string) (*ShareProperties, error) { - uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), url.Values{"restype": {"share"}}) + headers, err := f.getResourceHeaders(ToPathSegment(name), compNone, resourceShare, http.MethodHead) + if err != nil { + return nil, err + } + return &ShareProperties{ + LastModified: headers.Get("Last-Modified"), + Etag: headers.Get("Etag"), + Quota: headers.Get("x-ms-share-quota"), + }, nil +} - headers := f.client.getStandardHeaders() - resp, err := f.client.exec("HEAD", uri, headers, nil) +// returns HTTP header data for the specified directory or share +func (f FileServiceClient) getResourceHeaders(path string, comp compType, res resourceType, verb string) (http.Header, error) { + resp, err := f.getResourceNoClose(path, comp, res, verb, nil) if err != nil { return nil, err } defer resp.body.Close() - if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { + if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { return nil, err } - return &ShareProperties{ - LastModified: resp.headers.Get("Last-Modified"), - Etag: resp.headers.Get("Etag"), - Quota: resp.headers.Get("x-ms-share-quota"), - }, nil + return resp.headers, nil +} + +// gets the specified resource, doesn't close the response body +func (f FileServiceClient) getResourceNoClose(path string, comp compType, res resourceType, verb string, extraHeaders map[string]string) (*storageResponse, error) { + if err := f.checkForStorageEmulator(); err != nil { + return nil, err + } + + params := getURLInitValues(comp, res) + uri := f.client.getEndpoint(fileServiceName, path, params) + headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) + + return f.client.exec(verb, uri, headers, nil) +} + +// SetFileProperties operation sets system properties on the specified file. +// +// Some keys may be converted to Camel-Case before sending. All keys +// are returned in lower case by SetFileProperties. HTTP header names +// are case-insensitive so case munging should not matter to other +// applications either. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166975.aspx +func (f FileServiceClient) SetFileProperties(path string, props FileProperties) error { + return f.setResourceHeaders(path, compProperties, resourceFile, headersFromStruct(props)) } // SetShareProperties replaces the ShareHeaders for the specified file. @@ -201,26 +659,21 @@ func (f FileServiceClient) GetShareProperties(name string) (*ShareProperties, er // // See https://msdn.microsoft.com/en-us/library/azure/mt427368.aspx func (f FileServiceClient) SetShareProperties(name string, shareHeaders ShareHeaders) error { - params := url.Values{} - params.Set("restype", "share") - params.Set("comp", "properties") + return f.setResourceHeaders(ToPathSegment(name), compProperties, resourceShare, headersFromStruct(shareHeaders)) +} - uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), params) - headers := f.client.getStandardHeaders() +// DeleteDirectory operation removes the specified empty directory. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn166969.aspx +func (f FileServiceClient) DeleteDirectory(path string) error { + return f.deleteResource(path, resourceDirectory) +} - extraHeaders := headersFromStruct(shareHeaders) - - for k, v := range extraHeaders { - headers[k] = v - } - - resp, err := f.client.exec("PUT", uri, headers, nil) - if err != nil { - return err - } - defer resp.body.Close() - - return checkRespCode(resp.statusCode, []int{http.StatusOK}) +// DeleteFile operation immediately removes the file from the storage account. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn689085.aspx +func (f FileServiceClient) DeleteFile(path string) error { + return f.deleteResource(path, resourceFile) } // DeleteShare operation marks the specified share for deletion. The share @@ -229,12 +682,7 @@ func (f FileServiceClient) SetShareProperties(name string, shareHeaders ShareHea // // See https://msdn.microsoft.com/en-us/library/azure/dn689090.aspx func (f FileServiceClient) DeleteShare(name string) error { - resp, err := f.deleteShare(name) - if err != nil { - return err - } - defer resp.body.Close() - return checkRespCode(resp.statusCode, []int{http.StatusAccepted}) + return f.deleteResource(ToPathSegment(name), resourceShare) } // DeleteShareIfExists operation marks the specified share for deletion if it @@ -244,7 +692,7 @@ func (f FileServiceClient) DeleteShare(name string) error { // // See https://msdn.microsoft.com/en-us/library/azure/dn689090.aspx func (f FileServiceClient) DeleteShareIfExists(name string) (bool, error) { - resp, err := f.deleteShare(name) + resp, err := f.deleteResourceNoClose(ToPathSegment(name), resourceShare) if resp != nil { defer resp.body.Close() if resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound { @@ -254,14 +702,49 @@ func (f FileServiceClient) DeleteShareIfExists(name string) (bool, error) { return false, err } -// deleteShare makes the call to Delete Share operation endpoint and returns -// the response -func (f FileServiceClient) deleteShare(name string) (*storageResponse, error) { +// deletes the resource and returns the response +func (f FileServiceClient) deleteResource(path string, res resourceType) error { + resp, err := f.deleteResourceNoClose(path, res) + if err != nil { + return err + } + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusAccepted}) +} + +// deletes the resource and returns the response, doesn't close the response body +func (f FileServiceClient) deleteResourceNoClose(path string, res resourceType) (*storageResponse, error) { if err := f.checkForStorageEmulator(); err != nil { return nil, err } - uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), url.Values{"restype": {"share"}}) - return f.client.exec("DELETE", uri, f.client.getStandardHeaders(), nil) + + values := getURLInitValues(compNone, res) + uri := f.client.getEndpoint(fileServiceName, path, values) + return f.client.exec(http.MethodDelete, uri, f.client.getStandardHeaders(), nil) +} + +// SetDirectoryMetadata replaces the metadata for the specified directory. +// +// Some keys may be converted to Camel-Case before sending. All keys +// are returned in lower case by GetDirectoryMetadata. HTTP header names +// are case-insensitive so case munging should not matter to other +// applications either. +// +// See https://msdn.microsoft.com/en-us/library/azure/mt427370.aspx +func (f FileServiceClient) SetDirectoryMetadata(path string, metadata map[string]string) error { + return f.setResourceHeaders(path, compMetadata, resourceDirectory, mergeMDIntoExtraHeaders(metadata, nil)) +} + +// SetFileMetadata replaces the metadata for the specified file. +// +// Some keys may be converted to Camel-Case before sending. All keys +// are returned in lower case by GetFileMetadata. HTTP header names +// are case-insensitive so case munging should not matter to other +// applications either. +// +// See https://msdn.microsoft.com/en-us/library/azure/dn689097.aspx +func (f FileServiceClient) SetFileMetadata(path string, metadata map[string]string) error { + return f.setResourceHeaders(path, compMetadata, resourceFile, mergeMDIntoExtraHeaders(metadata, nil)) } // SetShareMetadata replaces the metadata for the specified Share. @@ -272,22 +755,43 @@ func (f FileServiceClient) deleteShare(name string) (*storageResponse, error) { // applications either. // // See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx -func (f FileServiceClient) SetShareMetadata(name string, metadata map[string]string, extraHeaders map[string]string) error { - params := url.Values{} - params.Set("restype", "share") - params.Set("comp", "metadata") +func (f FileServiceClient) SetShareMetadata(name string, metadata map[string]string) error { + return f.setResourceHeaders(ToPathSegment(name), compMetadata, resourceShare, mergeMDIntoExtraHeaders(metadata, nil)) +} - uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), params) - headers := f.client.getStandardHeaders() - for k, v := range metadata { - headers[userDefinedMetadataHeaderPrefix+k] = v +// merges metadata into extraHeaders and returns extraHeaders +func mergeMDIntoExtraHeaders(metadata, extraHeaders map[string]string) map[string]string { + if metadata == nil && extraHeaders == nil { + return nil } + if extraHeaders == nil { + extraHeaders = make(map[string]string) + } + for k, v := range metadata { + extraHeaders[userDefinedMetadataHeaderPrefix+k] = v + } + return extraHeaders +} +// merges extraHeaders into headers and returns headers +func mergeHeaders(headers, extraHeaders map[string]string) map[string]string { for k, v := range extraHeaders { headers[k] = v } + return headers +} - resp, err := f.client.exec("PUT", uri, headers, nil) +// sets extra header data for the specified resource +func (f FileServiceClient) setResourceHeaders(path string, comp compType, res resourceType, extraHeaders map[string]string) error { + if err := f.checkForStorageEmulator(); err != nil { + return err + } + + params := getURLInitValues(comp, res) + uri := f.client.getEndpoint(fileServiceName, path, params) + headers := mergeHeaders(f.client.getStandardHeaders(), extraHeaders) + + resp, err := f.client.exec(http.MethodPut, uri, headers, nil) if err != nil { return err } @@ -296,6 +800,26 @@ func (f FileServiceClient) SetShareMetadata(name string, metadata map[string]str return checkRespCode(resp.statusCode, []int{http.StatusOK}) } +// GetDirectoryMetadata returns all user-defined metadata for the specified directory. +// +// All metadata keys will be returned in lower case. (HTTP header +// names are case-insensitive.) +// +// See https://msdn.microsoft.com/en-us/library/azure/mt427371.aspx +func (f FileServiceClient) GetDirectoryMetadata(path string) (map[string]string, error) { + return f.getMetadata(path, resourceDirectory) +} + +// GetFileMetadata returns all user-defined metadata for the specified file. +// +// All metadata keys will be returned in lower case. (HTTP header +// names are case-insensitive.) +// +// See https://msdn.microsoft.com/en-us/library/azure/dn689098.aspx +func (f FileServiceClient) GetFileMetadata(path string) (map[string]string, error) { + return f.getMetadata(path, resourceFile) +} + // GetShareMetadata returns all user-defined metadata for the specified share. // // All metadata keys will be returned in lower case. (HTTP header @@ -303,25 +827,27 @@ func (f FileServiceClient) SetShareMetadata(name string, metadata map[string]str // // See https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx func (f FileServiceClient) GetShareMetadata(name string) (map[string]string, error) { - params := url.Values{} - params.Set("restype", "share") - params.Set("comp", "metadata") + return f.getMetadata(ToPathSegment(name), resourceShare) +} - uri := f.client.getEndpoint(fileServiceName, pathForFileShare(name), params) - headers := f.client.getStandardHeaders() +// gets metadata for the specified resource +func (f FileServiceClient) getMetadata(path string, res resourceType) (map[string]string, error) { + if err := f.checkForStorageEmulator(); err != nil { + return nil, err + } - resp, err := f.client.exec("GET", uri, headers, nil) + headers, err := f.getResourceHeaders(path, compMetadata, res, http.MethodGet) if err != nil { return nil, err } - defer resp.body.Close() - if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { - return nil, err - } + return getFileMDFromHeaders(headers), nil +} +// returns a map of custom metadata values from the specified HTTP header +func getFileMDFromHeaders(header http.Header) map[string]string { metadata := make(map[string]string) - for k, v := range resp.headers { + for k, v := range header { // Can't trust CanonicalHeaderKey() to munge case // reliably. "_" is allowed in identifiers: // https://msdn.microsoft.com/en-us/library/azure/dd179414.aspx @@ -339,7 +865,7 @@ func (f FileServiceClient) GetShareMetadata(name string) (map[string]string, err k = k[len(userDefinedMetadataHeaderPrefix):] metadata[k] = v[len(v)-1] } - return metadata, nil + return metadata } //checkForStorageEmulator determines if the client is setup for use with diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go index 3ecf4aca0..0cd357844 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/queue.go @@ -82,6 +82,24 @@ func (p PeekMessagesParameters) getParameters() url.Values { return out } +// UpdateMessageParameters is the set of options can be specified for Update Messsage +// operation. A zero struct does not use any preferences for the request. +type UpdateMessageParameters struct { + PopReceipt string + VisibilityTimeout int +} + +func (p UpdateMessageParameters) getParameters() url.Values { + out := url.Values{} + if p.PopReceipt != "" { + out.Set("popreceipt", p.PopReceipt) + } + if p.VisibilityTimeout != 0 { + out.Set("visibilitytimeout", strconv.Itoa(p.VisibilityTimeout)) + } + return out +} + // GetMessagesResponse represents a response returned from Get Messages // operation. type GetMessagesResponse struct { @@ -304,3 +322,23 @@ func (c QueueServiceClient) DeleteMessage(queue, messageID, popReceipt string) e defer resp.body.Close() return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) } + +// UpdateMessage operation deletes the specified message. +// +// See https://msdn.microsoft.com/en-us/library/azure/hh452234.aspx +func (c QueueServiceClient) UpdateMessage(queue string, messageID string, message string, params UpdateMessageParameters) error { + uri := c.client.getEndpoint(queueServiceName, pathForMessage(queue, messageID), params.getParameters()) + req := putMessageRequest{MessageText: message} + body, nn, err := xmlMarshal(req) + if err != nil { + return err + } + headers := c.client.getStandardHeaders() + headers["Content-Length"] = fmt.Sprintf("%d", nn) + resp, err := c.client.exec("PUT", uri, headers, body) + if err != nil { + return err + } + defer resp.body.Close() + return checkRespCode(resp.statusCode, []int{http.StatusNoContent}) +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/table_entities.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/table_entities.go index 1b5919cd1..a26d9c6f5 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/table_entities.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/table_entities.go @@ -10,6 +10,8 @@ import ( "reflect" ) +// Annotating as secure for gas scanning +/* #nosec */ const ( partitionKeyNode = "PartitionKey" rowKeyNode = "RowKey" diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go index d71c6ce55..57ca1b6d9 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go @@ -77,7 +77,7 @@ func headersFromStruct(v interface{}) map[string]string { for i := 0; i < value.NumField(); i++ { key := value.Type().Field(i).Tag.Get("header") val := value.Field(i).String() - if val != "" { + if key != "" && val != "" { headers[key] = val } } From 4549791522dd484c277e476fded2eb819add3341 Mon Sep 17 00:00:00 2001 From: Ahmet Alp Balkan Date: Fri, 16 Dec 2016 13:34:32 -0800 Subject: [PATCH 2/3] Update azure sdk version in vendor.conf Signed-off-by: Ahmet Alp Balkan --- vendor.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor.conf b/vendor.conf index 894530429..9ba2e7a1b 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,4 +1,4 @@ -github.com/Azure/azure-sdk-for-go/storage 0b5fe2abe0271ba07049eacaa65922d67c319543 +github.com/Azure/azure-sdk-for-go/storage c6f0533defaaaa26ea4dff3c9774e36033088112 github.com/Sirupsen/logrus d26492970760ca5d33129d2d799e34be5c4782eb github.com/aws/aws-sdk-go 90dec2183a5f5458ee79cbaf4b8e9ab910bc81a6 github.com/bshuster-repo/logrus-logstash-hook 5f729f2fb50a301153cae84ff5c58981d51c095a From cc68bdb38b69e969a0af7734b4af86f927b4bf4a Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 9 Jan 2017 17:27:52 -0800 Subject: [PATCH 3/3] Remove unneeded subpackage Signed-off-by: Derek McGowan (github: dmcgowan) --- vendor.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor.conf b/vendor.conf index 9ba2e7a1b..5f6306356 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,4 +1,4 @@ -github.com/Azure/azure-sdk-for-go/storage c6f0533defaaaa26ea4dff3c9774e36033088112 +github.com/Azure/azure-sdk-for-go c6f0533defaaaa26ea4dff3c9774e36033088112 github.com/Sirupsen/logrus d26492970760ca5d33129d2d799e34be5c4782eb github.com/aws/aws-sdk-go 90dec2183a5f5458ee79cbaf4b8e9ab910bc81a6 github.com/bshuster-repo/logrus-logstash-hook 5f729f2fb50a301153cae84ff5c58981d51c095a