// Copyright 2014 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package storage import ( "encoding/base64" "io" "sync" "time" "golang.org/x/net/context" raw "google.golang.org/api/storage/v1" ) // Bucket represents a Google Cloud Storage bucket. type Bucket struct { // Name is the name of the bucket. Name string // ACL is the list of access control rules on the bucket. ACL []ACLRule // DefaultObjectACL is the list of access controls to // apply to new objects when no object ACL is provided. DefaultObjectACL []ACLRule // Location is the location of the bucket. It defaults to "US". Location string // Metageneration is the metadata generation of the bucket. // Read-only. Metageneration int64 // StorageClass is the storage class of the bucket. This defines // how objects in the bucket are stored and determines the SLA // and the cost of storage. Typical values are "STANDARD" and // "DURABLE_REDUCED_AVAILABILITY". Defaults to "STANDARD". StorageClass string // Created is the creation time of the bucket. // Read-only. Created time.Time } func newBucket(b *raw.Bucket) *Bucket { if b == nil { return nil } bucket := &Bucket{ Name: b.Name, Location: b.Location, Metageneration: b.Metageneration, StorageClass: b.StorageClass, Created: convertTime(b.TimeCreated), } acl := make([]ACLRule, len(b.Acl)) for i, rule := range b.Acl { acl[i] = ACLRule{ Entity: ACLEntity(rule.Entity), Role: ACLRole(rule.Role), } } bucket.ACL = acl objACL := make([]ACLRule, len(b.DefaultObjectAcl)) for i, rule := range b.DefaultObjectAcl { objACL[i] = ACLRule{ Entity: ACLEntity(rule.Entity), Role: ACLRole(rule.Role), } } bucket.DefaultObjectACL = objACL return bucket } // ObjectAttrs is the user-editable object attributes. type ObjectAttrs struct { // Name is the name of the object. Name string // ContentType is the MIME type of the object's content. // Optional. ContentType string // ContentLanguage is the optional RFC 1766 Content-Language of // the object's content sent in response headers. ContentLanguage string // ContentEncoding is the optional Content-Encoding of the object // sent it the response headers. ContentEncoding string // CacheControl is the optional Cache-Control header of the object // sent in the response headers. CacheControl string // ContentDisposition is the optional Content-Disposition header of the object // sent in the response headers. ContentDisposition string // ACL is the list of access control rules for the object. // Optional. If nil or empty, existing ACL rules are preserved. ACL []ACLRule // Metadata represents user-provided metadata, in key/value pairs. // It can be nil if the current metadata values needs to preserved. Metadata map[string]string } func (o ObjectAttrs) toRawObject(bucket string) *raw.Object { var acl []*raw.ObjectAccessControl if len(o.ACL) > 0 { acl = make([]*raw.ObjectAccessControl, len(o.ACL)) for i, rule := range o.ACL { acl[i] = &raw.ObjectAccessControl{ Entity: string(rule.Entity), Role: string(rule.Role), } } } return &raw.Object{ Bucket: bucket, Name: o.Name, ContentType: o.ContentType, ContentEncoding: o.ContentEncoding, ContentLanguage: o.ContentLanguage, CacheControl: o.CacheControl, ContentDisposition: o.ContentDisposition, Acl: acl, Metadata: o.Metadata, } } // Object represents a Google Cloud Storage (GCS) object. type Object struct { // Bucket is the name of the bucket containing this GCS object. Bucket string // Name is the name of the object within the bucket. Name string // ContentType is the MIME type of the object's content. ContentType string // ContentLanguage is the content language of the object's content. ContentLanguage string // CacheControl is the Cache-Control header to be sent in the response // headers when serving the object data. CacheControl string // ACL is the list of access control rules for the object. ACL []ACLRule // Owner is the owner of the object. // // If non-zero, it is in the form of "user-". Owner string // Size is the length of the object's content. Size int64 // ContentEncoding is the encoding of the object's content. ContentEncoding string // MD5 is the MD5 hash of the object's content. MD5 []byte // CRC32C is the CRC32 checksum of the object's content using // the Castagnoli93 polynomial. CRC32C uint32 // MediaLink is an URL to the object's content. MediaLink string // Metadata represents user-provided metadata, in key/value pairs. // It can be nil if no metadata is provided. Metadata map[string]string // Generation is the generation number of the object's content. Generation int64 // MetaGeneration is the version of the metadata for this // object at this generation. This field is used for preconditions // and for detecting changes in metadata. A metageneration number // is only meaningful in the context of a particular generation // of a particular object. MetaGeneration int64 // StorageClass is the storage class of the bucket. // This value defines how objects in the bucket are stored and // determines the SLA and the cost of storage. Typical values are // "STANDARD" and "DURABLE_REDUCED_AVAILABILITY". // It defaults to "STANDARD". StorageClass string // Deleted is the time the object was deleted. // If not deleted, it is the zero value. Deleted time.Time // Updated is the creation or modification time of the object. // For buckets with versioning enabled, changing an object's // metadata does not change this property. Updated time.Time } // convertTime converts a time in RFC3339 format to time.Time. // If any error occurs in parsing, the zero-value time.Time is silently returned. func convertTime(t string) time.Time { var r time.Time if t != "" { r, _ = time.Parse(time.RFC3339, t) } return r } func newObject(o *raw.Object) *Object { if o == nil { return nil } acl := make([]ACLRule, len(o.Acl)) for i, rule := range o.Acl { acl[i] = ACLRule{ Entity: ACLEntity(rule.Entity), Role: ACLRole(rule.Role), } } owner := "" if o.Owner != nil { owner = o.Owner.Entity } md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash) var crc32c uint32 d, err := base64.StdEncoding.DecodeString(o.Crc32c) if err == nil && len(d) == 4 { crc32c = uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]) } return &Object{ Bucket: o.Bucket, Name: o.Name, ContentType: o.ContentType, ContentLanguage: o.ContentLanguage, CacheControl: o.CacheControl, ACL: acl, Owner: owner, ContentEncoding: o.ContentEncoding, Size: int64(o.Size), MD5: md5, CRC32C: crc32c, MediaLink: o.MediaLink, Metadata: o.Metadata, Generation: o.Generation, MetaGeneration: o.Metageneration, StorageClass: o.StorageClass, Deleted: convertTime(o.TimeDeleted), Updated: convertTime(o.Updated), } } // Query represents a query to filter objects from a bucket. type Query struct { // Delimiter returns results in a directory-like fashion. // Results will contain only objects whose names, aside from the // prefix, do not contain delimiter. Objects whose names, // aside from the prefix, contain delimiter will have their name, // truncated after the delimiter, returned in prefixes. // Duplicate prefixes are omitted. // Optional. Delimiter string // Prefix is the prefix filter to query objects // whose names begin with this prefix. // Optional. Prefix string // Versions indicates whether multiple versions of the same // object will be included in the results. Versions bool // Cursor is a previously-returned page token // representing part of the larger set of results to view. // Optional. Cursor string // MaxResults is the maximum number of items plus prefixes // to return. As duplicate prefixes are omitted, // fewer total results may be returned than requested. // The default page limit is used if it is negative or zero. MaxResults int } // Objects represents a list of objects returned from // a bucket look-p request and a query to retrieve more // objects from the next pages. type Objects struct { // Results represent a list of object results. Results []*Object // Next is the continuation query to retrieve more // results with the same filtering criteria. If there // are no more results to retrieve, it is nil. Next *Query // Prefixes represents prefixes of objects // matching-but-not-listed up to and including // the requested delimiter. Prefixes []string } // contentTyper implements ContentTyper to enable an // io.ReadCloser to specify its MIME type. type contentTyper struct { io.Reader t string } func (c *contentTyper) ContentType() string { return c.t } // A Writer writes a Cloud Storage object. type Writer struct { // ObjectAttrs are optional attributes to set on the object. Any attributes // must be initialized before the first Write call. Nil or zero-valued // attributes are ignored. ObjectAttrs ctx context.Context bucket string name string once sync.Once opened bool r io.Reader pw *io.PipeWriter donec chan struct{} // closed after err and obj are set. err error obj *Object } func (w *Writer) open() { attrs := w.ObjectAttrs // Always set the name, otherwise the backend // rejects the request and responds with an HTTP 400. if attrs.Name == "" { attrs.Name = w.name } pr, pw := io.Pipe() w.r = &contentTyper{pr, attrs.ContentType} w.pw = pw w.opened = true go func() { resp, err := rawService(w.ctx).Objects.Insert( w.bucket, attrs.toRawObject(w.bucket)).Media(w.r).Projection("full").Context(w.ctx).Do() w.err = err if err == nil { w.obj = newObject(resp) } else { pr.CloseWithError(w.err) } close(w.donec) }() } // Write appends to w. func (w *Writer) Write(p []byte) (n int, err error) { if w.err != nil { return 0, w.err } if !w.opened { w.open() } return w.pw.Write(p) } // Close completes the write operation and flushes any buffered data. // If Close doesn't return an error, metadata about the written object // can be retrieved by calling Object. func (w *Writer) Close() error { if !w.opened { w.open() } if err := w.pw.Close(); err != nil { return err } <-w.donec return w.err } // CloseWithError aborts the write operation with the provided error. // CloseWithError always returns nil. func (w *Writer) CloseWithError(err error) error { if !w.opened { return nil } return w.pw.CloseWithError(err) } // Object returns metadata about a successfully-written object. // It's only valid to call it after Close returns nil. func (w *Writer) Object() *Object { return w.obj }